@crawlee/basic 4.0.0-beta.28 → 4.0.0-beta.281

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +11 -2
  3. package/index.js +11 -1
  4. package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
  5. package/internals/autoscaling/autoscaled-pool.js +389 -0
  6. package/internals/autoscaling/concurrency-system.d.ts +235 -0
  7. package/internals/autoscaling/concurrency-system.js +357 -0
  8. package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
  9. package/internals/autoscaling/cpu-load-signal.js +46 -0
  10. package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
  11. package/internals/autoscaling/event-loop-load-signal.js +60 -0
  12. package/internals/autoscaling/index.d.ts +9 -0
  13. package/internals/autoscaling/index.js +9 -0
  14. package/internals/autoscaling/load-signal.d.ts +98 -0
  15. package/internals/autoscaling/load-signal.js +103 -0
  16. package/internals/autoscaling/memory-load-signal.d.ts +45 -0
  17. package/internals/autoscaling/memory-load-signal.js +104 -0
  18. package/internals/autoscaling/snapshotter.d.ts +84 -0
  19. package/internals/autoscaling/snapshotter.js +67 -0
  20. package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
  21. package/internals/autoscaling/storage-backend-load-signal.js +73 -0
  22. package/internals/autoscaling/system-status.d.ts +112 -0
  23. package/internals/autoscaling/system-status.js +139 -0
  24. package/internals/autoscaling/weighted-avg.d.ts +5 -0
  25. package/internals/autoscaling/weighted-avg.js +14 -0
  26. package/internals/basic-crawler.d.ts +370 -374
  27. package/internals/basic-crawler.js +1416 -857
  28. package/internals/cookie-utils.d.ts +40 -0
  29. package/internals/cookie-utils.js +122 -0
  30. package/internals/crawler-run.d.ts +76 -0
  31. package/internals/crawler-run.js +144 -0
  32. package/internals/crawlers/context-pipeline.d.ts +75 -0
  33. package/internals/crawlers/context-pipeline.js +109 -0
  34. package/internals/crawlers/crawler-commons.d.ts +216 -0
  35. package/internals/crawlers/crawler-commons.js +1 -0
  36. package/internals/crawlers/error-snapshotter.d.ts +29 -0
  37. package/internals/crawlers/error-snapshotter.js +116 -0
  38. package/internals/crawlers/error-tracker.d.ts +51 -0
  39. package/internals/crawlers/error-tracker.js +316 -0
  40. package/internals/crawlers/index.d.ts +4 -0
  41. package/internals/crawlers/index.js +3 -0
  42. package/internals/crawlers/internals/types.d.ts +7 -0
  43. package/internals/crawlers/internals/types.js +1 -0
  44. package/internals/crawlers/statistics.d.ts +326 -0
  45. package/internals/crawlers/statistics.js +559 -0
  46. package/internals/crawling-request.d.ts +86 -0
  47. package/internals/crawling-request.js +127 -0
  48. package/internals/enqueue-links/enqueue-links.d.ts +147 -0
  49. package/internals/enqueue-links/enqueue-links.js +35 -0
  50. package/internals/enqueue-links/index.d.ts +3 -0
  51. package/internals/enqueue-links/index.js +3 -0
  52. package/internals/enqueue-links/shared.d.ts +107 -0
  53. package/internals/enqueue-links/shared.js +228 -0
  54. package/internals/errors.d.ts +73 -0
  55. package/internals/errors.js +108 -0
  56. package/internals/http.d.ts +9 -0
  57. package/internals/http.js +28 -0
  58. package/internals/request-timeout.d.ts +54 -0
  59. package/internals/request-timeout.js +98 -0
  60. package/internals/router.d.ts +299 -0
  61. package/internals/router.js +312 -0
  62. package/internals/send-request.d.ts +4 -4
  63. package/internals/send-request.js +18 -8
  64. package/internals/session-pool/consts.d.ts +1 -0
  65. package/internals/session-pool/consts.js +1 -0
  66. package/internals/session-pool/errors.d.ts +7 -0
  67. package/internals/session-pool/errors.js +11 -0
  68. package/internals/session-pool/fingerprint.d.ts +9 -0
  69. package/internals/session-pool/fingerprint.js +30 -0
  70. package/internals/session-pool/index.d.ts +4 -0
  71. package/internals/session-pool/index.js +4 -0
  72. package/internals/session-pool/session-pool.d.ts +193 -0
  73. package/internals/session-pool/session-pool.js +453 -0
  74. package/internals/session-pool/session.d.ts +151 -0
  75. package/internals/session-pool/session.js +225 -0
  76. package/internals/sitemap-request-loader.d.ts +165 -0
  77. package/internals/sitemap-request-loader.js +492 -0
  78. package/internals/throttling-request-manager.d.ts +229 -0
  79. package/internals/throttling-request-manager.js +860 -0
  80. package/package.json +23 -17
  81. package/index.d.ts.map +0 -1
  82. package/index.js.map +0 -1
  83. package/internals/basic-crawler.d.ts.map +0 -1
  84. package/internals/basic-crawler.js.map +0 -1
  85. package/internals/send-request.d.ts.map +0 -1
  86. package/internals/send-request.js.map +0 -1
@@ -0,0 +1,492 @@
1
+ import { Transform } from 'node:stream';
2
+ import { EnqueueStrategy } from '@crawlee/utils';
3
+ import { parseArgument, parseSitemap, schemas } from '@crawlee/utils/internal';
4
+ import { z } from 'zod';
5
+ import { EventType, KeyValueStore, purgeDefaultStorages, Request, RequestManagerTandem, RequestQueue, serviceLocator, } from '@crawlee/core';
6
+ import { constructUrlPatternObjects, urlPatternSchema } from './enqueue-links/index.js';
7
+ const sitemapRequestLoaderOptionsSchema = z.strictObject({
8
+ sitemapUrls: schemas.arrayOf(z.string(), 'strings'),
9
+ proxyUrl: z.string().optional(),
10
+ persistStateKey: z.string().optional(),
11
+ signal: z.unknown().optional(),
12
+ timeoutMillis: schemas.anyNumber.optional(),
13
+ maxBufferSize: schemas.anyNumber.default(200),
14
+ enqueueStrategy: z.enum(EnqueueStrategy).default(EnqueueStrategy.SameHostname),
15
+ parseSitemapOptions: z.looseObject({}).optional(),
16
+ include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
17
+ exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
18
+ persistenceOptions: z.looseObject({}).optional(),
19
+ httpClient: schemas.httpClient.optional(),
20
+ concurrency: z.number().int().gte(1).default(1),
21
+ });
22
+ /** @internal */
23
+ const STATE_PERSISTENCE_KEY = 'SITEMAP_REQUEST_LOADER_STATE';
24
+ /**
25
+ * A list of URLs to crawl parsed from a sitemap.
26
+ *
27
+ * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.
28
+ */
29
+ export class SitemapRequestLoader {
30
+ /**
31
+ * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.
32
+ * @internal
33
+ */
34
+ inProgress = new Set();
35
+ /**
36
+ * Map of returned Request objects that have not been marked as handled yet.
37
+ *
38
+ * We use this to persist custom user fields on the in-progress requests.
39
+ */
40
+ #requestData = new Map();
41
+ /**
42
+ * Object for keeping track of the sitemap parsing progress.
43
+ */
44
+ #sitemapParsingProgress = {
45
+ inProgressSitemapUrls: new Set(),
46
+ /**
47
+ * URLs already emitted from each in-progress sitemap. Used for tracking partially loaded sitemaps across migrations.
48
+ */
49
+ inProgressEntriesPerSitemap: new Map(),
50
+ /**
51
+ * Set of sitemap URLs that have not been parsed yet. If this and `inProgressSitemapUrls` are empty, the sitemap loading is finished.
52
+ */
53
+ pendingSitemapUrls: new Set(),
54
+ /**
55
+ * Set of sitemap URLs that have already been parsed. Keeps sitemap indexes that reference each other from being loaded again.
56
+ */
57
+ processedSitemapUrls: new Set(),
58
+ };
59
+ /**
60
+ * Maximum number of sitemaps to load in parallel.
61
+ */
62
+ #concurrency;
63
+ /**
64
+ * Object stream of URLs parsed from the sitemaps.
65
+ * Using `highWaterMark`, this can manage the speed of the sitemap loading.
66
+ *
67
+ * Fetch the next URL to be processed using `fetchNextRequest()`.
68
+ */
69
+ #urlQueueStream;
70
+ /**
71
+ * Indicates whether the request list sitemap loading was aborted.
72
+ *
73
+ * If the loading was aborted before the sitemaps were fully loaded, the request list might be missing some URLs.
74
+ * The `isSitemapFullyLoaded` method can be used to check if the sitemaps were fully loaded.
75
+ *
76
+ * If the loading is aborted and all the requests are handled, `checkReadiness()` will report `finished`.
77
+ */
78
+ #abortLoading = false;
79
+ /** Number of URLs that were marked as handled */
80
+ #handledUrlCount = 0;
81
+ #persistStateKey;
82
+ #store;
83
+ #closed = false;
84
+ /**
85
+ * Proxy URL to be used for sitemap loading.
86
+ */
87
+ #proxyUrl;
88
+ /**
89
+ * Enqueue strategy applied to sitemap-derived URLs and stamped onto the emitted `Request` objects.
90
+ */
91
+ #enqueueStrategy;
92
+ /**
93
+ * Logger instance.
94
+ */
95
+ #log;
96
+ #urlExcludePatternObjects = [];
97
+ #urlPatternObjects = [];
98
+ /** EventManager used to handle persistence */
99
+ #events;
100
+ #persistenceOptions;
101
+ /** @internal */
102
+ constructor(options) {
103
+ const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls, enqueueStrategy, concurrency, } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
104
+ this.#log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
105
+ if (exclude?.length) {
106
+ this.#urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));
107
+ }
108
+ if (include?.length) {
109
+ this.#urlPatternObjects.push(...constructUrlPatternObjects(include));
110
+ }
111
+ this.#persistStateKey = persistStateKey;
112
+ this.#persistenceOptions = { enable: true, ...persistenceOptions };
113
+ this.#proxyUrl = proxyUrl;
114
+ this.#enqueueStrategy = enqueueStrategy;
115
+ this.#concurrency = concurrency;
116
+ this.#urlQueueStream = this.#createNewStream(maxBufferSize);
117
+ this.#sitemapParsingProgress.pendingSitemapUrls = new Set(sitemapUrls);
118
+ this.#events = serviceLocator.getEventManager();
119
+ this.persistState = this.persistState.bind(this);
120
+ }
121
+ /**
122
+ * Creates a new object stream with the specified highWaterMark.
123
+ * @param highWaterMark High water mark for the stream (the maximum number of objects the stream will buffer).
124
+ * @returns A new object stream.
125
+ */
126
+ #createNewStream(highWaterMark) {
127
+ return new Transform({
128
+ objectMode: true,
129
+ highWaterMark,
130
+ }).pause();
131
+ }
132
+ /**
133
+ * Checks whether the URL matches the `include` / `exclude` patterns provided in the `options`.
134
+ * @param url URL to be checked.
135
+ * @returns `true` if the URL matches the patterns, `false` otherwise.
136
+ */
137
+ #isUrlMatchingPatterns(url) {
138
+ return (!this.#urlExcludePatternObjects.some(({ match }) => match(url)) &&
139
+ (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(({ match }) => match(url))));
140
+ }
141
+ /**
142
+ * Adds a URL to the queue of parsed URLs.
143
+ *
144
+ * Blocks if the stream is full until it is drained.
145
+ */
146
+ async #pushNextUrl(url) {
147
+ return new Promise((resolve) => {
148
+ if (this.#closed || (url && !this.#isUrlMatchingPatterns(url))) {
149
+ resolve();
150
+ return;
151
+ }
152
+ if (!this.#urlQueueStream.push(url)) {
153
+ // This doesn't work with the 'drain' event (it's not emitted for some reason).
154
+ this.#urlQueueStream.once('readdata', () => {
155
+ resolve();
156
+ });
157
+ }
158
+ else {
159
+ resolve();
160
+ }
161
+ });
162
+ }
163
+ /**
164
+ * Reads the next URL from the queue of parsed URLs.
165
+ *
166
+ * If the stream is empty, blocks until a new URL is pushed.
167
+ * @returns The next URL from the queue or `null` if we have read all URLs.
168
+ */
169
+ async #readNextUrl() {
170
+ return new Promise((resolve) => {
171
+ if (this.#closed) {
172
+ resolve(null);
173
+ return;
174
+ }
175
+ const result = this.#urlQueueStream.read();
176
+ if (!result && !this.isSitemapFullyLoaded()) {
177
+ this.#urlQueueStream.once('readable', () => {
178
+ const nextUrl = this.#urlQueueStream.read();
179
+ resolve(nextUrl);
180
+ });
181
+ }
182
+ else {
183
+ resolve(result);
184
+ }
185
+ this.#urlQueueStream.emit('readdata');
186
+ });
187
+ }
188
+ /**
189
+ * Indicates whether the background processing of sitemap contents has successfully finished.
190
+ *
191
+ * If this is `false`, the background processing is either still in progress or was aborted.
192
+ */
193
+ isSitemapFullyLoaded() {
194
+ return (this.#sitemapParsingProgress.inProgressSitemapUrls.size === 0 &&
195
+ this.#sitemapParsingProgress.pendingSitemapUrls.size === 0);
196
+ }
197
+ #isUrlAlreadyEmitted(url) {
198
+ for (const entries of this.#sitemapParsingProgress.inProgressEntriesPerSitemap.values()) {
199
+ if (entries.has(url)) {
200
+ return true;
201
+ }
202
+ }
203
+ return false;
204
+ }
205
+ /**
206
+ * Processes a single sitemap URL and emits its URLs to the queue.
207
+ * Nested sitemaps are added to the pending queue for later processing.
208
+ */
209
+ async #loadSingleSitemap(sitemapUrl, parseSitemapOptions) {
210
+ const progress = this.#sitemapParsingProgress;
211
+ progress.inProgressSitemapUrls.add(sitemapUrl);
212
+ if (!progress.inProgressEntriesPerSitemap.has(sitemapUrl)) {
213
+ progress.inProgressEntriesPerSitemap.set(sitemapUrl, new Set());
214
+ }
215
+ const entries = progress.inProgressEntriesPerSitemap.get(sitemapUrl);
216
+ try {
217
+ for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.#proxyUrl, {
218
+ ...parseSitemapOptions,
219
+ maxDepth: 0,
220
+ emitNestedSitemaps: true,
221
+ enqueueStrategy: this.#enqueueStrategy,
222
+ })) {
223
+ // Leave the sitemap in progress so `isSitemapFullyLoaded()` keeps reporting the abort.
224
+ if (this.#abortLoading)
225
+ return;
226
+ if (!item.originSitemapUrl) {
227
+ // This is a nested sitemap
228
+ if (!progress.processedSitemapUrls.has(item.loc) && !progress.inProgressSitemapUrls.has(item.loc)) {
229
+ progress.pendingSitemapUrls.add(item.loc);
230
+ }
231
+ continue;
232
+ }
233
+ if (!entries.has(item.loc) && !this.#isUrlAlreadyEmitted(item.loc)) {
234
+ await this.#pushNextUrl(item.loc);
235
+ entries.add(item.loc);
236
+ }
237
+ }
238
+ }
239
+ catch (e) {
240
+ this.#log.error(`Error loading sitemap contents from ${sitemapUrl}:`, e);
241
+ }
242
+ progress.inProgressSitemapUrls.delete(sitemapUrl);
243
+ progress.pendingSitemapUrls.delete(sitemapUrl);
244
+ progress.inProgressEntriesPerSitemap.delete(sitemapUrl);
245
+ progress.processedSitemapUrls.add(sitemapUrl);
246
+ }
247
+ /**
248
+ * Start processing the sitemaps and loading the URLs.
249
+ *
250
+ * Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
251
+ */
252
+ async #load({ parseSitemapOptions, }) {
253
+ const activeLoaders = [];
254
+ const startNewLoaders = () => {
255
+ while (!this.#abortLoading &&
256
+ activeLoaders.length < this.#concurrency &&
257
+ this.#sitemapParsingProgress.pendingSitemapUrls.size > 0) {
258
+ const sitemapUrl = this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value;
259
+ this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
260
+ const loaderPromise = this.#loadSingleSitemap(sitemapUrl, parseSitemapOptions).finally(() => {
261
+ const index = activeLoaders.indexOf(loaderPromise);
262
+ if (index !== -1)
263
+ void activeLoaders.splice(index, 1);
264
+ });
265
+ activeLoaders.push(loaderPromise);
266
+ }
267
+ };
268
+ startNewLoaders();
269
+ while ((activeLoaders.length > 0 || this.#sitemapParsingProgress.pendingSitemapUrls.size > 0) &&
270
+ !this.#abortLoading) {
271
+ if (activeLoaders.length > 0)
272
+ await Promise.race(activeLoaders);
273
+ startNewLoaders();
274
+ }
275
+ if (activeLoaders.length > 0)
276
+ await Promise.all(activeLoaders);
277
+ this.#urlQueueStream.end();
278
+ }
279
+ /**
280
+ * Open a sitemap and start processing it.
281
+ *
282
+ * Resolves to a new instance of `SitemapRequestLoader`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
283
+ *
284
+ * Track the loading progress using the `isSitemapFullyLoaded` property.
285
+ */
286
+ static async open(options) {
287
+ const { httpClient, ...restOptions } = options;
288
+ const requestList = new SitemapRequestLoader({
289
+ ...restOptions,
290
+ persistStateKey: options.persistStateKey ?? STATE_PERSISTENCE_KEY,
291
+ });
292
+ await requestList.#restoreState();
293
+ void requestList.#load({
294
+ parseSitemapOptions: { logger: serviceLocator.getLogger(), ...options.parseSitemapOptions, httpClient },
295
+ });
296
+ if (requestList.#persistenceOptions.enable) {
297
+ requestList.#events.on(EventType.PERSIST_STATE, requestList.persistState);
298
+ }
299
+ options?.signal?.addEventListener('abort', () => {
300
+ requestList.#abortLoading = true;
301
+ });
302
+ if (options.timeoutMillis) {
303
+ setTimeout(() => {
304
+ requestList.#abortLoading = true;
305
+ }, options.timeoutMillis);
306
+ }
307
+ return requestList;
308
+ }
309
+ /**
310
+ * @inheritDoc
311
+ */
312
+ async getTotalCount() {
313
+ // Total known so far = not-yet-fetched (still buffered in the stream) + in-progress (fetched but not
314
+ // yet handled) + already handled.
315
+ return this.#urlQueueStream.readableLength + this.inProgress.size + this.#handledUrlCount;
316
+ }
317
+ /**
318
+ * @inheritDoc
319
+ */
320
+ async getPendingCount() {
321
+ // Pending = everything not yet handled = not-yet-fetched + in-progress.
322
+ return this.#urlQueueStream.readableLength + this.inProgress.size;
323
+ }
324
+ /**
325
+ * Combines this list with a request manager (a {@link RequestQueue} by default) into a
326
+ * {@link RequestManagerTandem}, allowing requests to be added and reclaimed while still
327
+ * being read from this list first.
328
+ */
329
+ async toTandem(requestManager) {
330
+ return new RequestManagerTandem(this, requestManager ?? (await RequestQueue.open()));
331
+ }
332
+ /**
333
+ * @inheritDoc
334
+ */
335
+ async checkReadiness() {
336
+ if (this.#urlQueueStream.readableLength > 0) {
337
+ return { status: 'ready' };
338
+ }
339
+ // The parser is still running, so more URLs may yet arrive - on no schedule, hence no `readyAt`.
340
+ if (!this.isSitemapFullyLoaded() && !this.#abortLoading) {
341
+ return { status: 'waiting' };
342
+ }
343
+ return this.inProgress.size === 0 ? { status: 'finished' } : { status: 'waiting' };
344
+ }
345
+ /**
346
+ * @inheritDoc
347
+ */
348
+ async getHandledCount() {
349
+ return this.#handledUrlCount;
350
+ }
351
+ /**
352
+ * @inheritDoc
353
+ */
354
+ async persistState() {
355
+ if (this.#persistStateKey === undefined) {
356
+ return;
357
+ }
358
+ this.#store ??= await KeyValueStore.open();
359
+ const urlQueue = [];
360
+ while (this.#urlQueueStream.readableLength > 0) {
361
+ const url = this.#urlQueueStream.read();
362
+ if (url === null) {
363
+ break;
364
+ }
365
+ urlQueue.push(url);
366
+ }
367
+ // Create a new stream, as we have read all the URLs from the current one.
368
+ // Pushing the urls back to the original stream might not be possible if it has been ended.
369
+ const previousStream = this.#urlQueueStream;
370
+ const newStream = this.#createNewStream(previousStream.readableHighWaterMark);
371
+ for (const url of urlQueue) {
372
+ newStream.push(url);
373
+ }
374
+ if (previousStream.writableEnded) {
375
+ newStream.end();
376
+ }
377
+ this.#urlQueueStream = newStream;
378
+ // A `pushNextUrl()` call may be blocked on backpressure, waiting for a `readdata` event on the
379
+ // previous stream. That event is only ever emitted by `readNextUrl()` on the current stream, so
380
+ // after the swap the waiter would never be notified and the background sitemap loading would hang.
381
+ // Re-emit `readdata` on the previous stream to release any such pending waiter (its URL has already
382
+ // been transferred to the new stream above).
383
+ previousStream.emit('readdata');
384
+ const inProgressEntriesPerSitemap = {};
385
+ for (const [sitemapUrl, entries] of this.#sitemapParsingProgress.inProgressEntriesPerSitemap) {
386
+ inProgressEntriesPerSitemap[sitemapUrl] = Array.from(entries);
387
+ }
388
+ await this.#store.setValue(this.#persistStateKey, {
389
+ sitemapParsingProgress: {
390
+ pendingSitemapUrls: Array.from(this.#sitemapParsingProgress.pendingSitemapUrls),
391
+ inProgressSitemapUrls: Array.from(this.#sitemapParsingProgress.inProgressSitemapUrls),
392
+ inProgressEntriesPerSitemap,
393
+ processedSitemapUrls: Array.from(this.#sitemapParsingProgress.processedSitemapUrls),
394
+ },
395
+ // Re-queue in-progress requests to the front so they are retried if the state is restored.
396
+ urlQueue: [...this.inProgress, ...urlQueue],
397
+ requestData: Array.from(this.#requestData.entries()),
398
+ abortLoading: this.#abortLoading,
399
+ closed: this.#closed,
400
+ });
401
+ }
402
+ async #restoreState() {
403
+ await purgeDefaultStorages({ onlyPurgeOnce: true });
404
+ if (this.#persistStateKey === undefined) {
405
+ return;
406
+ }
407
+ this.#store ??= await KeyValueStore.open();
408
+ const state = await this.#store.getValue(this.#persistStateKey);
409
+ if (state === null) {
410
+ return;
411
+ }
412
+ const progress = state.sitemapParsingProgress;
413
+ const inProgressSitemapUrls = new Set(progress.inProgressSitemapUrls ?? []);
414
+ if (progress.inProgressSitemapUrl && !progress.inProgressSitemapUrls) {
415
+ inProgressSitemapUrls.add(progress.inProgressSitemapUrl);
416
+ }
417
+ const inProgressEntriesPerSitemap = new Map();
418
+ if (progress.inProgressEntriesPerSitemap) {
419
+ for (const [sitemapUrl, entries] of Object.entries(progress.inProgressEntriesPerSitemap)) {
420
+ inProgressEntriesPerSitemap.set(sitemapUrl, new Set(entries));
421
+ }
422
+ }
423
+ else if (progress.inProgressEntries && progress.inProgressSitemapUrl) {
424
+ inProgressEntriesPerSitemap.set(progress.inProgressSitemapUrl, new Set(progress.inProgressEntries));
425
+ }
426
+ // Interrupted sitemaps are re-parsed; `inProgressEntriesPerSitemap` skips the URLs they already emitted.
427
+ this.#sitemapParsingProgress = {
428
+ pendingSitemapUrls: new Set([...progress.pendingSitemapUrls, ...inProgressSitemapUrls]),
429
+ inProgressSitemapUrls: new Set(),
430
+ inProgressEntriesPerSitemap,
431
+ processedSitemapUrls: new Set(progress.processedSitemapUrls ?? []),
432
+ };
433
+ this.#requestData = new Map(state.requestData ?? []);
434
+ for (const url of state.urlQueue) {
435
+ this.#urlQueueStream.push(url);
436
+ }
437
+ this.#abortLoading = state.abortLoading;
438
+ this.#closed = state.closed;
439
+ }
440
+ /**
441
+ * @inheritDoc
442
+ */
443
+ async fetchNextRequest() {
444
+ const nextUrl = await this.#readNextUrl();
445
+ if (!nextUrl) {
446
+ return null;
447
+ }
448
+ // A restored in-progress request already has its Request data; don't overwrite it.
449
+ if (!this.#requestData.has(nextUrl)) {
450
+ this.#requestData.set(nextUrl, new Request({ url: nextUrl, enqueueStrategy: this.#enqueueStrategy }));
451
+ }
452
+ this.inProgress.add(nextUrl);
453
+ return this.#requestData.get(nextUrl);
454
+ }
455
+ /**
456
+ * @inheritDoc
457
+ */
458
+ async *[Symbol.asyncIterator]() {
459
+ while ((await this.checkReadiness()).status !== 'finished') {
460
+ const request = await this.fetchNextRequest();
461
+ if (!request)
462
+ break;
463
+ yield request;
464
+ }
465
+ }
466
+ /**
467
+ * Aborts the internal sitemap loading, stops the processing of the sitemap contents and drops all the pending URLs.
468
+ *
469
+ * Calling `fetchNextRequest()` after this method will always return `null`.
470
+ */
471
+ async teardown() {
472
+ this.#closed = true;
473
+ this.#abortLoading = true;
474
+ this.#events.off(EventType.PERSIST_STATE, this.persistState);
475
+ await this.persistState();
476
+ this.#urlQueueStream.emit('readdata'); // unblocks the potentially waiting `pushNextUrl` call
477
+ }
478
+ /**
479
+ * @inheritDoc
480
+ */
481
+ async markRequestAsHandled(request) {
482
+ this.#handledUrlCount += 1;
483
+ this.#ensureInProgress(request.url);
484
+ this.inProgress.delete(request.url);
485
+ this.#requestData.delete(request.url);
486
+ }
487
+ #ensureInProgress(url) {
488
+ if (!this.inProgress.has(url)) {
489
+ throw new Error(`The request is not being processed (url: ${url})`);
490
+ }
491
+ }
492
+ }