@crawlee/basic 4.0.0-beta.269 → 4.0.0-beta.270

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -53,6 +53,10 @@ export class SitemapRequestLoader {
53
53
  * Set of sitemap URLs that have not been parsed yet. If the set is empty and `inProgressSitemapUrl` is `null`, the sitemap loading is finished.
54
54
  */
55
55
  pendingSitemapUrls: new Set(),
56
+ /**
57
+ * Set of sitemap URLs that have already been parsed. Keeps sitemap indexes that reference each other from being loaded again.
58
+ */
59
+ processedSitemapUrls: new Set(),
56
60
  };
57
61
  /**
58
62
  * Object stream of URLs parsed from the sitemaps.
@@ -205,7 +209,9 @@ export class SitemapRequestLoader {
205
209
  })) {
206
210
  if (!item.originSitemapUrl) {
207
211
  // This is a nested sitemap
208
- this.#sitemapParsingProgress.pendingSitemapUrls.add(item.loc);
212
+ if (!this.#sitemapParsingProgress.processedSitemapUrls.has(item.loc)) {
213
+ this.#sitemapParsingProgress.pendingSitemapUrls.add(item.loc);
214
+ }
209
215
  continue;
210
216
  }
211
217
  if (!this.#sitemapParsingProgress.inProgressEntries.has(item.loc)) {
@@ -218,6 +224,7 @@ export class SitemapRequestLoader {
218
224
  this.#log.error('Error loading sitemap contents:', e);
219
225
  }
220
226
  this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
227
+ this.#sitemapParsingProgress.processedSitemapUrls.add(sitemapUrl);
221
228
  this.#sitemapParsingProgress.inProgressEntries.clear();
222
229
  this.#sitemapParsingProgress.inProgressSitemapUrl = null;
223
230
  }
@@ -333,6 +340,7 @@ export class SitemapRequestLoader {
333
340
  pendingSitemapUrls: Array.from(this.#sitemapParsingProgress.pendingSitemapUrls),
334
341
  inProgressSitemapUrl: this.#sitemapParsingProgress.inProgressSitemapUrl,
335
342
  inProgressEntries: Array.from(this.#sitemapParsingProgress.inProgressEntries),
343
+ processedSitemapUrls: Array.from(this.#sitemapParsingProgress.processedSitemapUrls),
336
344
  },
337
345
  // Re-queue in-progress requests to the front so they are retried if the state is restored.
338
346
  urlQueue: [...this.inProgress, ...urlQueue],
@@ -355,6 +363,7 @@ export class SitemapRequestLoader {
355
363
  pendingSitemapUrls: new Set(state.sitemapParsingProgress.pendingSitemapUrls),
356
364
  inProgressSitemapUrl: state.sitemapParsingProgress.inProgressSitemapUrl,
357
365
  inProgressEntries: new Set(state.sitemapParsingProgress.inProgressEntries),
366
+ processedSitemapUrls: new Set(state.sitemapParsingProgress.processedSitemapUrls ?? []),
358
367
  };
359
368
  this.#requestData = new Map(state.requestData ?? []);
360
369
  for (const url of state.urlQueue) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-beta.269",
3
+ "version": "4.0.0-beta.270",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -42,10 +42,10 @@
42
42
  "@apify/datastructures": "^3.0.1",
43
43
  "@apify/timeout": "^1.0.1",
44
44
  "@apify/utilities": "^3.1.0",
45
- "@crawlee/core": "4.0.0-beta.269",
46
- "@crawlee/http-client": "4.0.0-beta.269",
47
- "@crawlee/types": "4.0.0-beta.269",
48
- "@crawlee/utils": "4.0.0-beta.269",
45
+ "@crawlee/core": "4.0.0-beta.270",
46
+ "@crawlee/http-client": "4.0.0-beta.270",
47
+ "@crawlee/types": "4.0.0-beta.270",
48
+ "@crawlee/utils": "4.0.0-beta.270",
49
49
  "@sapphire/async-queue": "^1.5.5",
50
50
  "@standard-schema/spec": "^1.1.0",
51
51
  "csv-stringify": "^6.9.0",
@@ -57,7 +57,7 @@
57
57
  "zod": "^4.6.7"
58
58
  },
59
59
  "optionalDependencies": {
60
- "@crawlee/impit-client": "4.0.0-beta.269"
60
+ "@crawlee/impit-client": "4.0.0-beta.270"
61
61
  },
62
62
  "lerna": {
63
63
  "command": {
@@ -66,5 +66,5 @@
66
66
  }
67
67
  }
68
68
  },
69
- "gitHead": "ee60f6c7d6aefb5a275a199a118244f75f4675e6"
69
+ "gitHead": "e251029ad24cab38129b61ea31fd430dba075f15"
70
70
  }