@crawlee/basic 4.0.0-beta.271 → 4.0.0-beta.273
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -99,7 +99,7 @@ export class CrawlingRequest extends Request {
|
|
|
99
99
|
message = omitStack
|
|
100
100
|
? errorOrMessage.message
|
|
101
101
|
: // .stack includes the message
|
|
102
|
-
errorOrMessage.stack;
|
|
102
|
+
(errorOrMessage.stack ?? errorOrMessage.message);
|
|
103
103
|
}
|
|
104
104
|
else if (Reflect.has(Object(errorOrMessage), 'message')) {
|
|
105
105
|
message = Reflect.get(Object(errorOrMessage), 'message');
|
|
@@ -80,6 +80,13 @@ export interface SitemapRequestLoaderOptions extends UrlConstraints {
|
|
|
80
80
|
* Custom HTTP client to be used for sitemap loading.
|
|
81
81
|
*/
|
|
82
82
|
httpClient?: BaseHttpClient;
|
|
83
|
+
/**
|
|
84
|
+
* Maximum number of sitemaps to load in parallel.
|
|
85
|
+
* This includes both the initial sitemaps and any nested sitemaps discovered during parsing.
|
|
86
|
+
*
|
|
87
|
+
* @default 1
|
|
88
|
+
*/
|
|
89
|
+
concurrency?: number;
|
|
83
90
|
}
|
|
84
91
|
/**
|
|
85
92
|
* A list of URLs to crawl parsed from a sitemap.
|
|
@@ -17,6 +17,7 @@ const sitemapRequestLoaderOptionsSchema = z.strictObject({
|
|
|
17
17
|
exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
18
18
|
persistenceOptions: z.looseObject({}).optional(),
|
|
19
19
|
httpClient: schemas.httpClient.optional(),
|
|
20
|
+
concurrency: z.number().int().gte(1).default(1),
|
|
20
21
|
});
|
|
21
22
|
/** @internal */
|
|
22
23
|
const STATE_PERSISTENCE_KEY = 'SITEMAP_REQUEST_LOADER_STATE';
|
|
@@ -41,16 +42,13 @@ export class SitemapRequestLoader {
|
|
|
41
42
|
* Object for keeping track of the sitemap parsing progress.
|
|
42
43
|
*/
|
|
43
44
|
#sitemapParsingProgress = {
|
|
45
|
+
inProgressSitemapUrls: new Set(),
|
|
44
46
|
/**
|
|
45
|
-
*
|
|
47
|
+
* URLs already emitted from each in-progress sitemap. Used for tracking partially loaded sitemaps across migrations.
|
|
46
48
|
*/
|
|
47
|
-
|
|
49
|
+
inProgressEntriesPerSitemap: new Map(),
|
|
48
50
|
/**
|
|
49
|
-
*
|
|
50
|
-
*/
|
|
51
|
-
inProgressEntries: new Set(),
|
|
52
|
-
/**
|
|
53
|
-
* Set of sitemap URLs that have not been parsed yet. If the set is empty and `inProgressSitemapUrl` is `null`, the sitemap loading is finished.
|
|
51
|
+
* Set of sitemap URLs that have not been parsed yet. If this and `inProgressSitemapUrls` are empty, the sitemap loading is finished.
|
|
54
52
|
*/
|
|
55
53
|
pendingSitemapUrls: new Set(),
|
|
56
54
|
/**
|
|
@@ -58,6 +56,10 @@ export class SitemapRequestLoader {
|
|
|
58
56
|
*/
|
|
59
57
|
processedSitemapUrls: new Set(),
|
|
60
58
|
};
|
|
59
|
+
/**
|
|
60
|
+
* Maximum number of sitemaps to load in parallel.
|
|
61
|
+
*/
|
|
62
|
+
#concurrency;
|
|
61
63
|
/**
|
|
62
64
|
* Object stream of URLs parsed from the sitemaps.
|
|
63
65
|
* Using `highWaterMark`, this can manage the speed of the sitemap loading.
|
|
@@ -98,7 +100,7 @@ export class SitemapRequestLoader {
|
|
|
98
100
|
#persistenceOptions;
|
|
99
101
|
/** @internal */
|
|
100
102
|
constructor(options) {
|
|
101
|
-
const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls, enqueueStrategy, } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
|
|
103
|
+
const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls, enqueueStrategy, concurrency, } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
|
|
102
104
|
this.#log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
|
|
103
105
|
if (exclude?.length) {
|
|
104
106
|
this.#urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));
|
|
@@ -110,6 +112,7 @@ export class SitemapRequestLoader {
|
|
|
110
112
|
this.#persistenceOptions = { enable: true, ...persistenceOptions };
|
|
111
113
|
this.#proxyUrl = proxyUrl;
|
|
112
114
|
this.#enqueueStrategy = enqueueStrategy;
|
|
115
|
+
this.#concurrency = concurrency;
|
|
113
116
|
this.#urlQueueStream = this.#createNewStream(maxBufferSize);
|
|
114
117
|
this.#sitemapParsingProgress.pendingSitemapUrls = new Set(sitemapUrls);
|
|
115
118
|
this.#events = serviceLocator.getEventManager();
|
|
@@ -188,46 +191,89 @@ export class SitemapRequestLoader {
|
|
|
188
191
|
* If this is `false`, the background processing is either still in progress or was aborted.
|
|
189
192
|
*/
|
|
190
193
|
isSitemapFullyLoaded() {
|
|
191
|
-
return (this.#sitemapParsingProgress.
|
|
194
|
+
return (this.#sitemapParsingProgress.inProgressSitemapUrls.size === 0 &&
|
|
192
195
|
this.#sitemapParsingProgress.pendingSitemapUrls.size === 0);
|
|
193
196
|
}
|
|
197
|
+
#isUrlAlreadyEmitted(url) {
|
|
198
|
+
for (const entries of this.#sitemapParsingProgress.inProgressEntriesPerSitemap.values()) {
|
|
199
|
+
if (entries.has(url)) {
|
|
200
|
+
return true;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return false;
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Processes a single sitemap URL and emits its URLs to the queue.
|
|
207
|
+
* Nested sitemaps are added to the pending queue for later processing.
|
|
208
|
+
*/
|
|
209
|
+
async #loadSingleSitemap(sitemapUrl, parseSitemapOptions) {
|
|
210
|
+
const progress = this.#sitemapParsingProgress;
|
|
211
|
+
progress.inProgressSitemapUrls.add(sitemapUrl);
|
|
212
|
+
if (!progress.inProgressEntriesPerSitemap.has(sitemapUrl)) {
|
|
213
|
+
progress.inProgressEntriesPerSitemap.set(sitemapUrl, new Set());
|
|
214
|
+
}
|
|
215
|
+
const entries = progress.inProgressEntriesPerSitemap.get(sitemapUrl);
|
|
216
|
+
try {
|
|
217
|
+
for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.#proxyUrl, {
|
|
218
|
+
...parseSitemapOptions,
|
|
219
|
+
maxDepth: 0,
|
|
220
|
+
emitNestedSitemaps: true,
|
|
221
|
+
enqueueStrategy: this.#enqueueStrategy,
|
|
222
|
+
})) {
|
|
223
|
+
// Leave the sitemap in progress so `isSitemapFullyLoaded()` keeps reporting the abort.
|
|
224
|
+
if (this.#abortLoading)
|
|
225
|
+
return;
|
|
226
|
+
if (!item.originSitemapUrl) {
|
|
227
|
+
// This is a nested sitemap
|
|
228
|
+
if (!progress.processedSitemapUrls.has(item.loc) && !progress.inProgressSitemapUrls.has(item.loc)) {
|
|
229
|
+
progress.pendingSitemapUrls.add(item.loc);
|
|
230
|
+
}
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
if (!entries.has(item.loc) && !this.#isUrlAlreadyEmitted(item.loc)) {
|
|
234
|
+
await this.#pushNextUrl(item.loc);
|
|
235
|
+
entries.add(item.loc);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
catch (e) {
|
|
240
|
+
this.#log.error(`Error loading sitemap contents from ${sitemapUrl}:`, e);
|
|
241
|
+
}
|
|
242
|
+
progress.inProgressSitemapUrls.delete(sitemapUrl);
|
|
243
|
+
progress.pendingSitemapUrls.delete(sitemapUrl);
|
|
244
|
+
progress.inProgressEntriesPerSitemap.delete(sitemapUrl);
|
|
245
|
+
progress.processedSitemapUrls.add(sitemapUrl);
|
|
246
|
+
}
|
|
194
247
|
/**
|
|
195
248
|
* Start processing the sitemaps and loading the URLs.
|
|
196
249
|
*
|
|
197
250
|
* Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
|
|
198
251
|
*/
|
|
199
252
|
async #load({ parseSitemapOptions, }) {
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
this.#sitemapParsingProgress.pendingSitemapUrls.add(item.loc);
|
|
214
|
-
}
|
|
215
|
-
continue;
|
|
216
|
-
}
|
|
217
|
-
if (!this.#sitemapParsingProgress.inProgressEntries.has(item.loc)) {
|
|
218
|
-
await this.#pushNextUrl(item.loc);
|
|
219
|
-
this.#sitemapParsingProgress.inProgressEntries.add(item.loc);
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
catch (e) {
|
|
224
|
-
this.#log.error('Error loading sitemap contents:', e);
|
|
253
|
+
const activeLoaders = [];
|
|
254
|
+
const startNewLoaders = () => {
|
|
255
|
+
while (!this.#abortLoading &&
|
|
256
|
+
activeLoaders.length < this.#concurrency &&
|
|
257
|
+
this.#sitemapParsingProgress.pendingSitemapUrls.size > 0) {
|
|
258
|
+
const sitemapUrl = this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value;
|
|
259
|
+
this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
|
|
260
|
+
const loaderPromise = this.#loadSingleSitemap(sitemapUrl, parseSitemapOptions).finally(() => {
|
|
261
|
+
const index = activeLoaders.indexOf(loaderPromise);
|
|
262
|
+
if (index !== -1)
|
|
263
|
+
void activeLoaders.splice(index, 1);
|
|
264
|
+
});
|
|
265
|
+
activeLoaders.push(loaderPromise);
|
|
225
266
|
}
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
this.#
|
|
267
|
+
};
|
|
268
|
+
startNewLoaders();
|
|
269
|
+
while ((activeLoaders.length > 0 || this.#sitemapParsingProgress.pendingSitemapUrls.size > 0) &&
|
|
270
|
+
!this.#abortLoading) {
|
|
271
|
+
if (activeLoaders.length > 0)
|
|
272
|
+
await Promise.race(activeLoaders);
|
|
273
|
+
startNewLoaders();
|
|
230
274
|
}
|
|
275
|
+
if (activeLoaders.length > 0)
|
|
276
|
+
await Promise.all(activeLoaders);
|
|
231
277
|
this.#urlQueueStream.end();
|
|
232
278
|
}
|
|
233
279
|
/**
|
|
@@ -335,11 +381,15 @@ export class SitemapRequestLoader {
|
|
|
335
381
|
// Re-emit `readdata` on the previous stream to release any such pending waiter (its URL has already
|
|
336
382
|
// been transferred to the new stream above).
|
|
337
383
|
previousStream.emit('readdata');
|
|
384
|
+
const inProgressEntriesPerSitemap = {};
|
|
385
|
+
for (const [sitemapUrl, entries] of this.#sitemapParsingProgress.inProgressEntriesPerSitemap) {
|
|
386
|
+
inProgressEntriesPerSitemap[sitemapUrl] = Array.from(entries);
|
|
387
|
+
}
|
|
338
388
|
await this.#store.setValue(this.#persistStateKey, {
|
|
339
389
|
sitemapParsingProgress: {
|
|
340
390
|
pendingSitemapUrls: Array.from(this.#sitemapParsingProgress.pendingSitemapUrls),
|
|
341
|
-
|
|
342
|
-
|
|
391
|
+
inProgressSitemapUrls: Array.from(this.#sitemapParsingProgress.inProgressSitemapUrls),
|
|
392
|
+
inProgressEntriesPerSitemap,
|
|
343
393
|
processedSitemapUrls: Array.from(this.#sitemapParsingProgress.processedSitemapUrls),
|
|
344
394
|
},
|
|
345
395
|
// Re-queue in-progress requests to the front so they are retried if the state is restored.
|
|
@@ -359,11 +409,26 @@ export class SitemapRequestLoader {
|
|
|
359
409
|
if (state === null) {
|
|
360
410
|
return;
|
|
361
411
|
}
|
|
412
|
+
const progress = state.sitemapParsingProgress;
|
|
413
|
+
const inProgressSitemapUrls = new Set(progress.inProgressSitemapUrls ?? []);
|
|
414
|
+
if (progress.inProgressSitemapUrl && !progress.inProgressSitemapUrls) {
|
|
415
|
+
inProgressSitemapUrls.add(progress.inProgressSitemapUrl);
|
|
416
|
+
}
|
|
417
|
+
const inProgressEntriesPerSitemap = new Map();
|
|
418
|
+
if (progress.inProgressEntriesPerSitemap) {
|
|
419
|
+
for (const [sitemapUrl, entries] of Object.entries(progress.inProgressEntriesPerSitemap)) {
|
|
420
|
+
inProgressEntriesPerSitemap.set(sitemapUrl, new Set(entries));
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
else if (progress.inProgressEntries && progress.inProgressSitemapUrl) {
|
|
424
|
+
inProgressEntriesPerSitemap.set(progress.inProgressSitemapUrl, new Set(progress.inProgressEntries));
|
|
425
|
+
}
|
|
426
|
+
// Interrupted sitemaps are re-parsed; `inProgressEntriesPerSitemap` skips the URLs they already emitted.
|
|
362
427
|
this.#sitemapParsingProgress = {
|
|
363
|
-
pendingSitemapUrls: new Set(
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
processedSitemapUrls: new Set(
|
|
428
|
+
pendingSitemapUrls: new Set([...progress.pendingSitemapUrls, ...inProgressSitemapUrls]),
|
|
429
|
+
inProgressSitemapUrls: new Set(),
|
|
430
|
+
inProgressEntriesPerSitemap,
|
|
431
|
+
processedSitemapUrls: new Set(progress.processedSitemapUrls ?? []),
|
|
367
432
|
};
|
|
368
433
|
this.#requestData = new Map(state.requestData ?? []);
|
|
369
434
|
for (const url of state.urlQueue) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.273",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.273",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.273",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.273",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.273",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.273"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "5ac532d43112438b53244a51bc76f0636429124a"
|
|
70
70
|
}
|