@crawlee/basic 4.0.0-beta.272 → 4.0.0-beta.273

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -80,6 +80,13 @@ export interface SitemapRequestLoaderOptions extends UrlConstraints {
80
80
  * Custom HTTP client to be used for sitemap loading.
81
81
  */
82
82
  httpClient?: BaseHttpClient;
83
+ /**
84
+ * Maximum number of sitemaps to load in parallel.
85
+ * This includes both the initial sitemaps and any nested sitemaps discovered during parsing.
86
+ *
87
+ * @default 1
88
+ */
89
+ concurrency?: number;
83
90
  }
84
91
  /**
85
92
  * A list of URLs to crawl parsed from a sitemap.
@@ -17,6 +17,7 @@ const sitemapRequestLoaderOptionsSchema = z.strictObject({
17
17
  exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
18
18
  persistenceOptions: z.looseObject({}).optional(),
19
19
  httpClient: schemas.httpClient.optional(),
20
+ concurrency: z.number().int().gte(1).default(1),
20
21
  });
21
22
  /** @internal */
22
23
  const STATE_PERSISTENCE_KEY = 'SITEMAP_REQUEST_LOADER_STATE';
@@ -41,16 +42,13 @@ export class SitemapRequestLoader {
41
42
  * Object for keeping track of the sitemap parsing progress.
42
43
  */
43
44
  #sitemapParsingProgress = {
45
+ inProgressSitemapUrls: new Set(),
44
46
  /**
45
- * URL of the sitemap that is currently being parsed. `null` if no sitemap is being parsed.
47
+ * URLs already emitted from each in-progress sitemap. Used for tracking partially loaded sitemaps across migrations.
46
48
  */
47
- inProgressSitemapUrl: null,
49
+ inProgressEntriesPerSitemap: new Map(),
48
50
  /**
49
- * Buffer for URLs from the currently parsed sitemap. Used for tracking partially loaded sitemaps across migrations.
50
- */
51
- inProgressEntries: new Set(),
52
- /**
53
- * Set of sitemap URLs that have not been parsed yet. If the set is empty and `inProgressSitemapUrl` is `null`, the sitemap loading is finished.
51
+ * Set of sitemap URLs that have not been parsed yet. If this and `inProgressSitemapUrls` are empty, the sitemap loading is finished.
54
52
  */
55
53
  pendingSitemapUrls: new Set(),
56
54
  /**
@@ -58,6 +56,10 @@ export class SitemapRequestLoader {
58
56
  */
59
57
  processedSitemapUrls: new Set(),
60
58
  };
59
+ /**
60
+ * Maximum number of sitemaps to load in parallel.
61
+ */
62
+ #concurrency;
61
63
  /**
62
64
  * Object stream of URLs parsed from the sitemaps.
63
65
  * Using `highWaterMark`, this can manage the speed of the sitemap loading.
@@ -98,7 +100,7 @@ export class SitemapRequestLoader {
98
100
  #persistenceOptions;
99
101
  /** @internal */
100
102
  constructor(options) {
101
- const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls, enqueueStrategy, } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
103
+ const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls, enqueueStrategy, concurrency, } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
102
104
  this.#log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
103
105
  if (exclude?.length) {
104
106
  this.#urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));
@@ -110,6 +112,7 @@ export class SitemapRequestLoader {
110
112
  this.#persistenceOptions = { enable: true, ...persistenceOptions };
111
113
  this.#proxyUrl = proxyUrl;
112
114
  this.#enqueueStrategy = enqueueStrategy;
115
+ this.#concurrency = concurrency;
113
116
  this.#urlQueueStream = this.#createNewStream(maxBufferSize);
114
117
  this.#sitemapParsingProgress.pendingSitemapUrls = new Set(sitemapUrls);
115
118
  this.#events = serviceLocator.getEventManager();
@@ -188,46 +191,89 @@ export class SitemapRequestLoader {
188
191
  * If this is `false`, the background processing is either still in progress or was aborted.
189
192
  */
190
193
  isSitemapFullyLoaded() {
191
- return (this.#sitemapParsingProgress.inProgressSitemapUrl === null &&
194
+ return (this.#sitemapParsingProgress.inProgressSitemapUrls.size === 0 &&
192
195
  this.#sitemapParsingProgress.pendingSitemapUrls.size === 0);
193
196
  }
197
+ #isUrlAlreadyEmitted(url) {
198
+ for (const entries of this.#sitemapParsingProgress.inProgressEntriesPerSitemap.values()) {
199
+ if (entries.has(url)) {
200
+ return true;
201
+ }
202
+ }
203
+ return false;
204
+ }
205
+ /**
206
+ * Processes a single sitemap URL and emits its URLs to the queue.
207
+ * Nested sitemaps are added to the pending queue for later processing.
208
+ */
209
+ async #loadSingleSitemap(sitemapUrl, parseSitemapOptions) {
210
+ const progress = this.#sitemapParsingProgress;
211
+ progress.inProgressSitemapUrls.add(sitemapUrl);
212
+ if (!progress.inProgressEntriesPerSitemap.has(sitemapUrl)) {
213
+ progress.inProgressEntriesPerSitemap.set(sitemapUrl, new Set());
214
+ }
215
+ const entries = progress.inProgressEntriesPerSitemap.get(sitemapUrl);
216
+ try {
217
+ for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.#proxyUrl, {
218
+ ...parseSitemapOptions,
219
+ maxDepth: 0,
220
+ emitNestedSitemaps: true,
221
+ enqueueStrategy: this.#enqueueStrategy,
222
+ })) {
223
+ // Leave the sitemap in progress so `isSitemapFullyLoaded()` keeps reporting the abort.
224
+ if (this.#abortLoading)
225
+ return;
226
+ if (!item.originSitemapUrl) {
227
+ // This is a nested sitemap
228
+ if (!progress.processedSitemapUrls.has(item.loc) && !progress.inProgressSitemapUrls.has(item.loc)) {
229
+ progress.pendingSitemapUrls.add(item.loc);
230
+ }
231
+ continue;
232
+ }
233
+ if (!entries.has(item.loc) && !this.#isUrlAlreadyEmitted(item.loc)) {
234
+ await this.#pushNextUrl(item.loc);
235
+ entries.add(item.loc);
236
+ }
237
+ }
238
+ }
239
+ catch (e) {
240
+ this.#log.error(`Error loading sitemap contents from ${sitemapUrl}:`, e);
241
+ }
242
+ progress.inProgressSitemapUrls.delete(sitemapUrl);
243
+ progress.pendingSitemapUrls.delete(sitemapUrl);
244
+ progress.inProgressEntriesPerSitemap.delete(sitemapUrl);
245
+ progress.processedSitemapUrls.add(sitemapUrl);
246
+ }
194
247
  /**
195
248
  * Start processing the sitemaps and loading the URLs.
196
249
  *
197
250
  * Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
198
251
  */
199
252
  async #load({ parseSitemapOptions, }) {
200
- while (!this.isSitemapFullyLoaded() && !this.#abortLoading) {
201
- const sitemapUrl = this.#sitemapParsingProgress.inProgressSitemapUrl ??
202
- this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value;
203
- try {
204
- for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.#proxyUrl, {
205
- ...parseSitemapOptions,
206
- maxDepth: 0,
207
- emitNestedSitemaps: true,
208
- enqueueStrategy: this.#enqueueStrategy,
209
- })) {
210
- if (!item.originSitemapUrl) {
211
- // This is a nested sitemap
212
- if (!this.#sitemapParsingProgress.processedSitemapUrls.has(item.loc)) {
213
- this.#sitemapParsingProgress.pendingSitemapUrls.add(item.loc);
214
- }
215
- continue;
216
- }
217
- if (!this.#sitemapParsingProgress.inProgressEntries.has(item.loc)) {
218
- await this.#pushNextUrl(item.loc);
219
- this.#sitemapParsingProgress.inProgressEntries.add(item.loc);
220
- }
221
- }
222
- }
223
- catch (e) {
224
- this.#log.error('Error loading sitemap contents:', e);
253
+ const activeLoaders = [];
254
+ const startNewLoaders = () => {
255
+ while (!this.#abortLoading &&
256
+ activeLoaders.length < this.#concurrency &&
257
+ this.#sitemapParsingProgress.pendingSitemapUrls.size > 0) {
258
+ const sitemapUrl = this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value;
259
+ this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
260
+ const loaderPromise = this.#loadSingleSitemap(sitemapUrl, parseSitemapOptions).finally(() => {
261
+ const index = activeLoaders.indexOf(loaderPromise);
262
+ if (index !== -1)
263
+ void activeLoaders.splice(index, 1);
264
+ });
265
+ activeLoaders.push(loaderPromise);
225
266
  }
226
- this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
227
- this.#sitemapParsingProgress.processedSitemapUrls.add(sitemapUrl);
228
- this.#sitemapParsingProgress.inProgressEntries.clear();
229
- this.#sitemapParsingProgress.inProgressSitemapUrl = null;
267
+ };
268
+ startNewLoaders();
269
+ while ((activeLoaders.length > 0 || this.#sitemapParsingProgress.pendingSitemapUrls.size > 0) &&
270
+ !this.#abortLoading) {
271
+ if (activeLoaders.length > 0)
272
+ await Promise.race(activeLoaders);
273
+ startNewLoaders();
230
274
  }
275
+ if (activeLoaders.length > 0)
276
+ await Promise.all(activeLoaders);
231
277
  this.#urlQueueStream.end();
232
278
  }
233
279
  /**
@@ -335,11 +381,15 @@ export class SitemapRequestLoader {
335
381
  // Re-emit `readdata` on the previous stream to release any such pending waiter (its URL has already
336
382
  // been transferred to the new stream above).
337
383
  previousStream.emit('readdata');
384
+ const inProgressEntriesPerSitemap = {};
385
+ for (const [sitemapUrl, entries] of this.#sitemapParsingProgress.inProgressEntriesPerSitemap) {
386
+ inProgressEntriesPerSitemap[sitemapUrl] = Array.from(entries);
387
+ }
338
388
  await this.#store.setValue(this.#persistStateKey, {
339
389
  sitemapParsingProgress: {
340
390
  pendingSitemapUrls: Array.from(this.#sitemapParsingProgress.pendingSitemapUrls),
341
- inProgressSitemapUrl: this.#sitemapParsingProgress.inProgressSitemapUrl,
342
- inProgressEntries: Array.from(this.#sitemapParsingProgress.inProgressEntries),
391
+ inProgressSitemapUrls: Array.from(this.#sitemapParsingProgress.inProgressSitemapUrls),
392
+ inProgressEntriesPerSitemap,
343
393
  processedSitemapUrls: Array.from(this.#sitemapParsingProgress.processedSitemapUrls),
344
394
  },
345
395
  // Re-queue in-progress requests to the front so they are retried if the state is restored.
@@ -359,11 +409,26 @@ export class SitemapRequestLoader {
359
409
  if (state === null) {
360
410
  return;
361
411
  }
412
+ const progress = state.sitemapParsingProgress;
413
+ const inProgressSitemapUrls = new Set(progress.inProgressSitemapUrls ?? []);
414
+ if (progress.inProgressSitemapUrl && !progress.inProgressSitemapUrls) {
415
+ inProgressSitemapUrls.add(progress.inProgressSitemapUrl);
416
+ }
417
+ const inProgressEntriesPerSitemap = new Map();
418
+ if (progress.inProgressEntriesPerSitemap) {
419
+ for (const [sitemapUrl, entries] of Object.entries(progress.inProgressEntriesPerSitemap)) {
420
+ inProgressEntriesPerSitemap.set(sitemapUrl, new Set(entries));
421
+ }
422
+ }
423
+ else if (progress.inProgressEntries && progress.inProgressSitemapUrl) {
424
+ inProgressEntriesPerSitemap.set(progress.inProgressSitemapUrl, new Set(progress.inProgressEntries));
425
+ }
426
+ // Interrupted sitemaps are re-parsed; `inProgressEntriesPerSitemap` skips the URLs they already emitted.
362
427
  this.#sitemapParsingProgress = {
363
- pendingSitemapUrls: new Set(state.sitemapParsingProgress.pendingSitemapUrls),
364
- inProgressSitemapUrl: state.sitemapParsingProgress.inProgressSitemapUrl,
365
- inProgressEntries: new Set(state.sitemapParsingProgress.inProgressEntries),
366
- processedSitemapUrls: new Set(state.sitemapParsingProgress.processedSitemapUrls ?? []),
428
+ pendingSitemapUrls: new Set([...progress.pendingSitemapUrls, ...inProgressSitemapUrls]),
429
+ inProgressSitemapUrls: new Set(),
430
+ inProgressEntriesPerSitemap,
431
+ processedSitemapUrls: new Set(progress.processedSitemapUrls ?? []),
367
432
  };
368
433
  this.#requestData = new Map(state.requestData ?? []);
369
434
  for (const url of state.urlQueue) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-beta.272",
3
+ "version": "4.0.0-beta.273",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -42,10 +42,10 @@
42
42
  "@apify/datastructures": "^3.0.1",
43
43
  "@apify/timeout": "^1.0.1",
44
44
  "@apify/utilities": "^3.1.0",
45
- "@crawlee/core": "4.0.0-beta.272",
46
- "@crawlee/http-client": "4.0.0-beta.272",
47
- "@crawlee/types": "4.0.0-beta.272",
48
- "@crawlee/utils": "4.0.0-beta.272",
45
+ "@crawlee/core": "4.0.0-beta.273",
46
+ "@crawlee/http-client": "4.0.0-beta.273",
47
+ "@crawlee/types": "4.0.0-beta.273",
48
+ "@crawlee/utils": "4.0.0-beta.273",
49
49
  "@sapphire/async-queue": "^1.5.5",
50
50
  "@standard-schema/spec": "^1.1.0",
51
51
  "csv-stringify": "^6.9.0",
@@ -57,7 +57,7 @@
57
57
  "zod": "^4.6.7"
58
58
  },
59
59
  "optionalDependencies": {
60
- "@crawlee/impit-client": "4.0.0-beta.272"
60
+ "@crawlee/impit-client": "4.0.0-beta.273"
61
61
  },
62
62
  "lerna": {
63
63
  "command": {
@@ -66,5 +66,5 @@
66
66
  }
67
67
  }
68
68
  },
69
- "gitHead": "31a2e8840172268cdef997686e1c22ee226a300f"
69
+ "gitHead": "5ac532d43112438b53244a51bc76f0636429124a"
70
70
  }