@crawlee/basic 4.0.0-beta.269 → 4.0.0-beta.270
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -53,6 +53,10 @@ export class SitemapRequestLoader {
|
|
|
53
53
|
* Set of sitemap URLs that have not been parsed yet. If the set is empty and `inProgressSitemapUrl` is `null`, the sitemap loading is finished.
|
|
54
54
|
*/
|
|
55
55
|
pendingSitemapUrls: new Set(),
|
|
56
|
+
/**
|
|
57
|
+
* Set of sitemap URLs that have already been parsed. Keeps sitemap indexes that reference each other from being loaded again.
|
|
58
|
+
*/
|
|
59
|
+
processedSitemapUrls: new Set(),
|
|
56
60
|
};
|
|
57
61
|
/**
|
|
58
62
|
* Object stream of URLs parsed from the sitemaps.
|
|
@@ -205,7 +209,9 @@ export class SitemapRequestLoader {
|
|
|
205
209
|
})) {
|
|
206
210
|
if (!item.originSitemapUrl) {
|
|
207
211
|
// This is a nested sitemap
|
|
208
|
-
this.#sitemapParsingProgress.
|
|
212
|
+
if (!this.#sitemapParsingProgress.processedSitemapUrls.has(item.loc)) {
|
|
213
|
+
this.#sitemapParsingProgress.pendingSitemapUrls.add(item.loc);
|
|
214
|
+
}
|
|
209
215
|
continue;
|
|
210
216
|
}
|
|
211
217
|
if (!this.#sitemapParsingProgress.inProgressEntries.has(item.loc)) {
|
|
@@ -218,6 +224,7 @@ export class SitemapRequestLoader {
|
|
|
218
224
|
this.#log.error('Error loading sitemap contents:', e);
|
|
219
225
|
}
|
|
220
226
|
this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
|
|
227
|
+
this.#sitemapParsingProgress.processedSitemapUrls.add(sitemapUrl);
|
|
221
228
|
this.#sitemapParsingProgress.inProgressEntries.clear();
|
|
222
229
|
this.#sitemapParsingProgress.inProgressSitemapUrl = null;
|
|
223
230
|
}
|
|
@@ -333,6 +340,7 @@ export class SitemapRequestLoader {
|
|
|
333
340
|
pendingSitemapUrls: Array.from(this.#sitemapParsingProgress.pendingSitemapUrls),
|
|
334
341
|
inProgressSitemapUrl: this.#sitemapParsingProgress.inProgressSitemapUrl,
|
|
335
342
|
inProgressEntries: Array.from(this.#sitemapParsingProgress.inProgressEntries),
|
|
343
|
+
processedSitemapUrls: Array.from(this.#sitemapParsingProgress.processedSitemapUrls),
|
|
336
344
|
},
|
|
337
345
|
// Re-queue in-progress requests to the front so they are retried if the state is restored.
|
|
338
346
|
urlQueue: [...this.inProgress, ...urlQueue],
|
|
@@ -355,6 +363,7 @@ export class SitemapRequestLoader {
|
|
|
355
363
|
pendingSitemapUrls: new Set(state.sitemapParsingProgress.pendingSitemapUrls),
|
|
356
364
|
inProgressSitemapUrl: state.sitemapParsingProgress.inProgressSitemapUrl,
|
|
357
365
|
inProgressEntries: new Set(state.sitemapParsingProgress.inProgressEntries),
|
|
366
|
+
processedSitemapUrls: new Set(state.sitemapParsingProgress.processedSitemapUrls ?? []),
|
|
358
367
|
};
|
|
359
368
|
this.#requestData = new Map(state.requestData ?? []);
|
|
360
369
|
for (const url of state.urlQueue) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.270",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.270",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.270",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.270",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.270",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.270"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "e251029ad24cab38129b61ea31fd430dba075f15"
|
|
70
70
|
}
|