@crawlee/utils 4.0.0-beta.25 → 4.0.0-beta.251

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +10 -17
  3. package/index.js +8 -16
  4. package/internal.d.ts +11 -0
  5. package/internal.js +10 -0
  6. package/internals/blocked.d.ts +0 -1
  7. package/internals/blocked.js +0 -1
  8. package/internals/cheerio.d.ts +1 -3
  9. package/internals/cheerio.js +25 -16
  10. package/internals/extract-microdata.d.ts +24 -0
  11. package/internals/extract-microdata.js +118 -0
  12. package/internals/extract-urls.d.ts +1 -2
  13. package/internals/extract-urls.js +18 -17
  14. package/internals/general.d.ts +0 -25
  15. package/internals/general.js +2 -110
  16. package/internals/iterables.d.ts +0 -80
  17. package/internals/iterables.js +0 -126
  18. package/internals/mutation-journal.d.ts +29 -0
  19. package/internals/mutation-journal.js +173 -0
  20. package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
  21. package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
  22. package/internals/robots.d.ts +35 -15
  23. package/internals/robots.js +66 -30
  24. package/internals/schemas.d.ts +114 -0
  25. package/internals/schemas.js +115 -0
  26. package/internals/sitemap.d.ts +44 -3
  27. package/internals/sitemap.js +171 -85
  28. package/internals/social.d.ts +2 -3
  29. package/internals/social.js +8 -6
  30. package/internals/url.d.ts +69 -1
  31. package/internals/url.js +120 -2
  32. package/internals/validation.d.ts +9 -0
  33. package/internals/validation.js +16 -0
  34. package/package.json +16 -15
  35. package/index.d.ts.map +0 -1
  36. package/index.js.map +0 -1
  37. package/internals/blocked.d.ts.map +0 -1
  38. package/internals/blocked.js.map +0 -1
  39. package/internals/cheerio.d.ts.map +0 -1
  40. package/internals/cheerio.js.map +0 -1
  41. package/internals/chunk.d.ts +0 -2
  42. package/internals/chunk.d.ts.map +0 -1
  43. package/internals/chunk.js +0 -40
  44. package/internals/chunk.js.map +0 -1
  45. package/internals/debug.d.ts +0 -32
  46. package/internals/debug.d.ts.map +0 -1
  47. package/internals/debug.js +0 -41
  48. package/internals/debug.js.map +0 -1
  49. package/internals/extract-urls.d.ts.map +0 -1
  50. package/internals/extract-urls.js.map +0 -1
  51. package/internals/general.d.ts.map +0 -1
  52. package/internals/general.js.map +0 -1
  53. package/internals/iterables.d.ts.map +0 -1
  54. package/internals/iterables.js.map +0 -1
  55. package/internals/open_graph_parser.d.ts.map +0 -1
  56. package/internals/open_graph_parser.js.map +0 -1
  57. package/internals/robots.d.ts.map +0 -1
  58. package/internals/robots.js.map +0 -1
  59. package/internals/sitemap.d.ts.map +0 -1
  60. package/internals/sitemap.js.map +0 -1
  61. package/internals/social.d.ts.map +0 -1
  62. package/internals/social.js.map +0 -1
  63. package/internals/system-info/cpu-info.d.ts +0 -64
  64. package/internals/system-info/cpu-info.d.ts.map +0 -1
  65. package/internals/system-info/cpu-info.js +0 -211
  66. package/internals/system-info/cpu-info.js.map +0 -1
  67. package/internals/system-info/memory-info.d.ts +0 -28
  68. package/internals/system-info/memory-info.d.ts.map +0 -1
  69. package/internals/system-info/memory-info.js +0 -118
  70. package/internals/system-info/memory-info.js.map +0 -1
  71. package/internals/system-info/ps-tree.d.ts +0 -18
  72. package/internals/system-info/ps-tree.d.ts.map +0 -1
  73. package/internals/system-info/ps-tree.js +0 -145
  74. package/internals/system-info/ps-tree.js.map +0 -1
  75. package/internals/typedefs.d.ts +0 -5
  76. package/internals/typedefs.d.ts.map +0 -1
  77. package/internals/typedefs.js +0 -9
  78. package/internals/typedefs.js.map +0 -1
  79. package/internals/url.d.ts.map +0 -1
  80. package/internals/url.js.map +0 -1
@@ -3,32 +3,30 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
3
3
  import { StringDecoder } from 'node:string_decoder';
4
4
  import { createGunzip } from 'node:zlib';
5
5
  import { FetchHttpClient } from '@crawlee/http-client';
6
- import { fileTypeStream } from 'file-type';
7
- import sax from 'sax';
8
- import MIMEType from 'whatwg-mimetype';
9
- import log from '@apify/log';
6
+ import { MIMEType } from 'whatwg-mimetype';
10
7
  import { mergeAsyncIterables } from './iterables.js';
11
- import { RobotsFile } from './robots.js';
8
+ import { RobotsTxtFile } from './robots.js';
9
+ import { filterUrl } from './url.js';
12
10
  class SitemapTxtParser extends Transform {
13
- decoder = new StringDecoder('utf8');
14
- buffer = '';
11
+ #decoder = new StringDecoder('utf8');
12
+ #buffer = '';
15
13
  constructor() {
16
14
  super({
17
15
  readableObjectMode: true,
18
16
  transform: (chunk, _encoding, callback) => {
19
- this.processBuffer(this.decoder.write(chunk), false);
17
+ this.#processBuffer(this.#decoder.write(chunk), false);
20
18
  callback();
21
19
  },
22
20
  flush: (callback) => {
23
- this.processBuffer(this.decoder.end(), true);
21
+ this.#processBuffer(this.#decoder.end(), true);
24
22
  callback();
25
23
  },
26
24
  });
27
25
  }
28
- processBuffer(input, finalize) {
29
- this.buffer += input;
30
- if (finalize || this.buffer.includes('\n')) {
31
- const parts = this.buffer
26
+ #processBuffer(input, finalize) {
27
+ this.#buffer += input;
28
+ if (finalize || this.#buffer.includes('\n')) {
29
+ const parts = this.#buffer
32
30
  .split('\n')
33
31
  .map((part) => part.trim())
34
32
  .filter((part) => part.length > 0);
@@ -36,99 +34,113 @@ class SitemapTxtParser extends Transform {
36
34
  for (const url of parts) {
37
35
  this.push({ type: 'url', loc: url });
38
36
  }
39
- this.buffer = '';
37
+ this.#buffer = '';
40
38
  }
41
39
  else if (parts.length > 0) {
42
40
  for (const url of parts.slice(0, -1)) {
43
41
  this.push({ type: 'url', loc: url });
44
42
  }
45
- this.buffer = parts.at(-1);
43
+ this.#buffer = parts.at(-1);
46
44
  }
47
45
  }
48
46
  }
49
47
  }
50
48
  class SitemapXmlParser extends Transform {
51
- decoder = new StringDecoder('utf8');
52
- parser = new sax.SAXParser(true);
53
- rootTagName;
54
- currentTag = undefined;
55
- url = {};
56
- constructor() {
49
+ #decoder = new StringDecoder('utf8');
50
+ #parser;
51
+ #rootTagName;
52
+ #currentTag = undefined;
53
+ #url = {};
54
+ static async create() {
55
+ const { SAXParser } = (await import('sax')).default;
56
+ return new SitemapXmlParser(new SAXParser(true));
57
+ }
58
+ constructor(parser) {
57
59
  super({
58
60
  readableObjectMode: true,
59
61
  transform: (chunk, _encoding, callback) => {
60
- this.parser.write(this.decoder.write(chunk));
62
+ this.#parser.write(this.#decoder.write(chunk));
61
63
  callback();
62
64
  },
63
65
  flush: (callback) => {
64
- const rest = this.decoder.end();
66
+ const rest = this.#decoder.end();
65
67
  if (rest.length > 0) {
66
- this.parser.write(rest);
68
+ this.#parser.write(rest);
67
69
  }
68
- this.parser.end();
70
+ this.#parser.end();
69
71
  callback();
70
72
  },
71
73
  });
72
- this.parser.onopentag = this.onOpenTag.bind(this);
73
- this.parser.onclosetag = this.onCloseTag.bind(this);
74
- this.parser.ontext = this.onText.bind(this);
75
- this.parser.oncdata = this.onText.bind(this);
76
- this.parser.onerror = this.destroy.bind(this);
74
+ this.#parser = parser;
75
+ this.#parser.onopentag = this.#onOpenTag.bind(this);
76
+ this.#parser.onclosetag = this.#onCloseTag.bind(this);
77
+ this.#parser.ontext = this.#onText.bind(this);
78
+ this.#parser.oncdata = this.#onText.bind(this);
79
+ this.#parser.onerror = this.destroy.bind(this);
77
80
  }
78
- onOpenTag(node) {
79
- if (this.rootTagName !== undefined) {
81
+ #onOpenTag(node) {
82
+ if (this.#rootTagName !== undefined) {
80
83
  if (node.name === 'loc' ||
81
84
  node.name === 'lastmod' ||
82
85
  node.name === 'priority' ||
83
86
  node.name === 'changefreq') {
84
- this.currentTag = node.name;
87
+ this.#currentTag = node.name;
85
88
  }
86
89
  }
87
90
  if (node.name === 'urlset') {
88
- this.rootTagName = 'urlset';
91
+ this.#rootTagName = 'urlset';
89
92
  }
90
93
  if (node.name === 'sitemapindex') {
91
- this.rootTagName = 'sitemapindex';
94
+ this.#rootTagName = 'sitemapindex';
92
95
  }
93
96
  }
94
- onCloseTag(name) {
97
+ #onCloseTag(name) {
95
98
  if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
96
- this.currentTag = undefined;
99
+ this.#currentTag = undefined;
97
100
  }
98
- if (name === 'url' && this.url.loc !== undefined) {
99
- this.push({ type: 'url', ...this.url, loc: this.url.loc });
100
- this.url = {};
101
+ if (name === 'url') {
102
+ if (this.#url.loc !== undefined) {
103
+ this.push({ type: 'url', ...this.#url, loc: this.#url.loc });
104
+ }
105
+ this.#url = {};
101
106
  }
102
107
  }
103
- onText(text) {
104
- if (this.currentTag === 'loc') {
105
- if (this.rootTagName === 'sitemapindex') {
106
- this.push({ type: 'sitemapUrl', url: text.trim() });
108
+ #onText(text) {
109
+ text = text.trim();
110
+ // Whitespace around a CDATA section arrives as separate text events.
111
+ if (text === '') {
112
+ return;
113
+ }
114
+ if (this.#currentTag === 'loc') {
115
+ if (this.#rootTagName === 'sitemapindex') {
116
+ this.push({ type: 'sitemapUrl', url: text });
107
117
  }
108
- if (this.rootTagName === 'urlset') {
109
- this.url ??= {};
110
- this.url.loc = text.trim();
118
+ if (this.#rootTagName === 'urlset') {
119
+ this.#url ??= {};
120
+ this.#url.loc = text;
111
121
  }
112
122
  }
113
- text = text.trim();
114
- if (this.currentTag === 'lastmod') {
115
- this.url.lastmod = new Date(text);
123
+ if (this.#currentTag === 'lastmod') {
124
+ const lastmod = new Date(text);
125
+ if (!Number.isNaN(lastmod.getTime())) {
126
+ this.#url.lastmod = lastmod;
127
+ }
116
128
  }
117
- if (this.currentTag === 'priority') {
118
- this.url.priority = Number(text);
129
+ if (this.#currentTag === 'priority') {
130
+ this.#url.priority = Number(text);
119
131
  }
120
- if (this.currentTag === 'changefreq') {
132
+ if (this.#currentTag === 'changefreq') {
121
133
  if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
122
- this.url.changefreq = text;
134
+ this.#url.changefreq = text;
123
135
  }
124
136
  }
125
137
  }
126
138
  }
127
139
  export async function* parseSitemap(initialSources, proxyUrl, options) {
128
- const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, } = options ?? {};
140
+ const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, enqueueStrategy = 'same-hostname', logger, } = options ?? {};
129
141
  const sources = [...initialSources];
130
142
  const visitedSitemapUrls = new Set();
131
- const createParser = (contentType = '', url) => {
143
+ const createParser = async (contentType = '', url) => {
132
144
  let mimeType;
133
145
  try {
134
146
  mimeType = new MIMEType(contentType);
@@ -137,7 +149,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
137
149
  mimeType = null;
138
150
  }
139
151
  if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
140
- return new SitemapXmlParser();
152
+ return SitemapXmlParser.create();
141
153
  }
142
154
  if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
143
155
  return new SitemapTxtParser();
@@ -147,12 +159,13 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
147
159
  while (sources.length > 0) {
148
160
  const source = sources.shift();
149
161
  if ((source?.depth ?? 0) > maxDepth) {
150
- log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
151
162
  continue;
152
163
  }
153
164
  let items = null;
165
+ // Parent URL, parsed once and reused as the origin for the strategy checks below.
166
+ let sitemapUrl;
154
167
  if (source.type === 'url') {
155
- const sitemapUrl = new URL(source.url);
168
+ sitemapUrl = new URL(source.url);
156
169
  visitedSitemapUrls.add(sitemapUrl.toString());
157
170
  let retriesLeft = sitemapRetries + 1;
158
171
  while (retriesLeft-- > 0) {
@@ -162,11 +175,11 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
162
175
  sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
163
176
  method: 'GET',
164
177
  headers: {
165
- accept: 'text/plain, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8',
178
+ accept: '*/*',
166
179
  },
167
180
  }), {
168
181
  proxyUrl,
169
- timeout,
182
+ timeoutMillis: timeout,
170
183
  });
171
184
  }
172
185
  catch (error) {
@@ -178,6 +191,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
178
191
  if (sitemapResponse.body === null) {
179
192
  break;
180
193
  }
194
+ const { fileTypeStream } = await import('file-type');
181
195
  const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
182
196
  if (streamWithType.fileType !== undefined) {
183
197
  contentType = streamWithType.fileType.mime;
@@ -191,7 +205,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
191
205
  sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
192
206
  }
193
207
  }
194
- items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
208
+ items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), await createParser(contentType ?? undefined, sitemapUrl), (e) => {
195
209
  if (e !== undefined && e !== null) {
196
210
  error = { type: 'parser', error: e };
197
211
  }
@@ -214,28 +228,53 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
214
228
  }
215
229
  }
216
230
  catch (e) {
217
- log.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
231
+ logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
218
232
  }
219
233
  }
220
234
  }
221
235
  else if (source.type === 'raw') {
222
- items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
236
+ items = pipeline(Readable.from([source.content]), await createParser('text/xml'), (error) => {
223
237
  if (error !== undefined) {
224
- log.warning(`Malformed sitemap content: ${error}`);
238
+ logger?.warning(`Malformed sitemap content: ${error}`);
225
239
  }
226
240
  });
227
241
  }
228
242
  if (items === null) {
229
243
  continue;
230
244
  }
245
+ // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
246
+ // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
247
+ let droppedUrlEntries = 0;
231
248
  for await (const item of items) {
232
249
  if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
250
+ if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
251
+ logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
252
+ continue;
253
+ }
254
+ // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
255
+ // parent. Raw string sources have no parent URL, so the check is skipped.
256
+ if (source.type === 'url') {
257
+ const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
258
+ if (!allowed) {
259
+ logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
260
+ continue;
261
+ }
262
+ }
233
263
  sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
234
264
  if (emitNestedSitemaps) {
235
265
  yield { loc: item.url, originSitemapUrl: null };
236
266
  }
237
267
  }
238
268
  if (item.type === 'url') {
269
+ // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
270
+ if (source.type === 'url') {
271
+ const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
272
+ if (!allowed) {
273
+ droppedUrlEntries++;
274
+ logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
275
+ continue;
276
+ }
277
+ }
239
278
  yield {
240
279
  ...item,
241
280
  originSitemapUrl: source.type === 'url'
@@ -244,6 +283,9 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
244
283
  };
245
284
  }
246
285
  }
286
+ if (droppedUrlEntries > 0 && source.type === 'url') {
287
+ logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
288
+ }
247
289
  }
248
290
  }
249
291
  /**
@@ -285,7 +327,7 @@ export class Sitemap {
285
327
  * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
286
328
  */
287
329
  static async load(urls, proxyUrl, parseSitemapOptions) {
288
- return await this.parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
330
+ return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
289
331
  }
290
332
  /**
291
333
  * Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
@@ -293,16 +335,17 @@ export class Sitemap {
293
335
  * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
294
336
  */
295
337
  static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
296
- return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
338
+ return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
297
339
  }
298
- static async parse(sources, proxyUrl, parseSitemapOptions) {
340
+ static async #parse(sources, proxyUrl, parseSitemapOptions) {
299
341
  const urls = [];
300
342
  try {
301
343
  for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
302
344
  urls.push(item.loc);
303
345
  }
304
346
  }
305
- catch {
347
+ catch (e) {
348
+ parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
306
349
  return new Sitemap([]);
307
350
  }
308
351
  return new Sitemap(urls);
@@ -316,7 +359,19 @@ export class Sitemap {
316
359
  * @returns An async iterable with the discovered sitemap URLs.
317
360
  */
318
361
  export async function* discoverValidSitemaps(urls, options = {}) {
319
- const { proxyUrl, httpClient = new FetchHttpClient() } = options;
362
+ const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
363
+ const controller = new AbortController();
364
+ const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
365
+ const onExternalAbort = () => controller.abort();
366
+ if (externalSignal) {
367
+ if (externalSignal.aborted) {
368
+ controller.abort();
369
+ }
370
+ else {
371
+ externalSignal.addEventListener('abort', onExternalAbort, { once: true });
372
+ }
373
+ }
374
+ const signal = controller.signal;
320
375
  const sitemapUrls = new Set();
321
376
  const addSitemapUrl = (url) => {
322
377
  const sizeBefore = sitemapUrls.size;
@@ -331,7 +386,11 @@ export async function* discoverValidSitemaps(urls, options = {}) {
331
386
  return false;
332
387
  }
333
388
  try {
334
- const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), { proxyUrl });
389
+ const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
390
+ proxyUrl,
391
+ timeoutMillis: requestTimeoutMillis,
392
+ signal,
393
+ });
335
394
  return response.status >= 200 && response.status < 400;
336
395
  }
337
396
  catch {
@@ -343,17 +402,24 @@ export async function* discoverValidSitemaps(urls, options = {}) {
343
402
  return;
344
403
  }
345
404
  try {
346
- const robotsFile = await RobotsFile.find(domainUrls[0], { proxyUrl, httpClient });
347
- for (const sitemapUrl of robotsFile.getSitemaps()) {
405
+ const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
406
+ proxyUrl,
407
+ timeoutMillis: requestTimeoutMillis,
408
+ signal,
409
+ httpClient,
410
+ logger,
411
+ });
412
+ // Surface all referenced sitemaps, including cross-host; scoping happens at load time.
413
+ for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
348
414
  if (addSitemapUrl(sitemapUrl)) {
349
415
  yield sitemapUrl;
350
416
  }
351
417
  }
352
418
  }
353
419
  catch (err) {
354
- log.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
420
+ logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
355
421
  }
356
- const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
422
+ const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));
357
423
  if (sitemapUrl !== undefined) {
358
424
  if (addSitemapUrl(sitemapUrl)) {
359
425
  yield sitemapUrl;
@@ -361,14 +427,29 @@ export async function* discoverValidSitemaps(urls, options = {}) {
361
427
  }
362
428
  else {
363
429
  const firstUrl = new URL(domainUrls[0]);
430
+ // The guessed candidates are well-known paths on the same origin, so the query string
431
+ // and fragment of the input URL are not part of them - carrying them over probes (and
432
+ // yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
433
+ firstUrl.search = '';
434
+ firstUrl.hash = '';
364
435
  const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
365
- for (const pathname of possibleSitemapPathnames) {
436
+ const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
366
437
  firstUrl.pathname = pathname;
367
- if (await urlExists(firstUrl.toString())) {
368
- if (addSitemapUrl(firstUrl.toString())) {
369
- yield firstUrl.toString();
438
+ return firstUrl.toString();
439
+ });
440
+ const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
441
+ for (const [index, result] of candidateResults.entries()) {
442
+ const candidateSitemapUrl = candidateSitemapUrls[index];
443
+ if (result.status === 'fulfilled') {
444
+ if (result.value && addSitemapUrl(candidateSitemapUrl)) {
445
+ yield candidateSitemapUrl;
370
446
  }
371
447
  }
448
+ else {
449
+ logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
450
+ error: result.reason,
451
+ });
452
+ }
372
453
  }
373
454
  }
374
455
  };
@@ -380,12 +461,17 @@ export async function* discoverValidSitemaps(urls, options = {}) {
380
461
  }, {});
381
462
  const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
382
463
  const discoveredUrls = new Set();
383
- for await (const url of mergeAsyncIterables(...iterables)) {
384
- if (discoveredUrls.has(url)) {
385
- continue;
464
+ try {
465
+ for await (const url of mergeAsyncIterables(...iterables)) {
466
+ if (discoveredUrls.has(url)) {
467
+ continue;
468
+ }
469
+ discoveredUrls.add(url);
470
+ yield url;
386
471
  }
387
- discoveredUrls.add(url);
388
- yield url;
472
+ }
473
+ finally {
474
+ clearTimeout(timeoutHandle);
475
+ externalSignal?.removeEventListener('abort', onExternalAbort);
389
476
  }
390
477
  }
391
- //# sourceMappingURL=sitemap.js.map
@@ -225,7 +225,7 @@ export declare const TWITTER_REGEX: RegExp;
225
225
  * ```
226
226
  * import { social } from 'crawlee';
227
227
  *
228
- * const matches = text.match(social.TWITTER_REGEX_STRING);
228
+ * const matches = text.match(social.TWITTER_REGEX_GLOBAL);
229
229
  * if (matches) console.log(`${matches.length} Twitter profiles found!`);
230
230
  * ```
231
231
  */
@@ -488,5 +488,4 @@ export declare const DISCORD_REGEX_GLOBAL: RegExp;
488
488
  * so that the caller doesn't need to parse the HTML document again, if needed.
489
489
  * @return An object with the social handles.
490
490
  */
491
- export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles;
492
- //# sourceMappingURL=social.d.ts.map
491
+ export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;
@@ -1,7 +1,9 @@
1
- import * as cheerio from 'cheerio';
2
1
  import { htmlToText } from './cheerio.js';
3
2
  // Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
4
- const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
3
+ // The dot-atom local part and domain labels use RFC 5321 length bounds ({1,64}, {0,62})
4
+ // instead of unbounded quantifiers to avoid quadratic backtracking (ReDoS) on long
5
+ // dotted or hyphenated inputs, e.g. text scraped by parseHandlesFromHtml().
6
+ const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}){0,32}|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
5
7
  /**
6
8
  * Regular expression to exactly match a single email address.
7
9
  * It has the following form: `/^...$/i`.
@@ -324,7 +326,7 @@ export const TWITTER_REGEX = new RegExp(`^${TWITTER_REGEX_STRING}$`, 'i');
324
326
  * ```
325
327
  * import { social } from 'crawlee';
326
328
  *
327
- * const matches = text.match(social.TWITTER_REGEX_STRING);
329
+ * const matches = text.match(social.TWITTER_REGEX_GLOBAL);
328
330
  * if (matches) console.log(`${matches.length} Twitter profiles found!`);
329
331
  * ```
330
332
  */
@@ -587,7 +589,8 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
587
589
  * so that the caller doesn't need to parse the HTML document again, if needed.
588
590
  * @return An object with the social handles.
589
591
  */
590
- export function parseHandlesFromHtml(html, data = null) {
592
+ export async function parseHandlesFromHtml(html, data = null) {
593
+ const cheerio = await import('cheerio/slim');
591
594
  const result = {
592
595
  emails: [],
593
596
  phones: [],
@@ -606,7 +609,7 @@ export function parseHandlesFromHtml(html, data = null) {
606
609
  const $ = cheerio.load(html, { xml: { decodeEntities: true } });
607
610
  if (data)
608
611
  data.$ = $;
609
- const text = htmlToText($);
612
+ const text = await htmlToText($);
610
613
  if (data)
611
614
  data.text = text;
612
615
  // NOTE: we need to parse each text separately, orherwise we might concatenate unrelated texts
@@ -644,4 +647,3 @@ export function parseHandlesFromHtml(html, data = null) {
644
647
  }
645
648
  return result;
646
649
  }
647
- //# sourceMappingURL=social.js.map
@@ -1,4 +1,73 @@
1
1
  import type { SearchParams } from '@crawlee/types';
2
+ /**
3
+ * The different enqueueing strategies available.
4
+ *
5
+ * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
6
+ *
7
+ * ```md
8
+ * Protocol Domain
9
+ * ┌────┐ ┌─────────┐
10
+ * https://example.crawlee.dev/...
11
+ * │ └─────────────────┤
12
+ * │ Hostname │
13
+ * │ │
14
+ * └─────────────────────────┘
15
+ * Origin
16
+ *```
17
+ *
18
+ * - The `Protocol` is usually `http` or `https`
19
+ * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
20
+ * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
21
+ * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
22
+ */
23
+ export declare enum EnqueueStrategy {
24
+ /**
25
+ * Matches any URLs found
26
+ */
27
+ All = "all",
28
+ /**
29
+ * Matches any URLs that have the same hostname.
30
+ * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
31
+ * `https://example.com/hello` will not be matched.
32
+ *
33
+ * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
34
+ */
35
+ SameHostname = "same-hostname",
36
+ /**
37
+ * Matches any URLs that have the same domain as the base URL.
38
+ * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
39
+ * `https://example.com`.
40
+ *
41
+ * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
42
+ */
43
+ SameDomain = "same-domain",
44
+ /**
45
+ * Matches any URLs that have the same hostname and protocol.
46
+ * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
47
+ * `http://wow.example.com/hello` will not be matched.
48
+ *
49
+ * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
50
+ */
51
+ SameOrigin = "same-origin"
52
+ }
53
+ /** Reusable suffix for log messages explaining why a non-`http(s)` URL was rejected. */
54
+ export declare const UNSUPPORTED_SCHEME_MESSAGE = "unsupported URL scheme (only http and https are allowed)";
55
+ /**
56
+ * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
57
+ * considered here (use {@link filterUrl} for the combined scheme + strategy check).
58
+ *
59
+ * The `enqueueLinks` implementation in `@crawlee/basic` matches the same strategies via glob patterns
60
+ * (see `packages/basic-crawler/src/internals/enqueue-links/enqueue-links.ts`) — keep the two in sync when changing either.
61
+ */
62
+ export declare function matchesEnqueueStrategy(strategy: EnqueueStrategy | `${EnqueueStrategy}`, target: URL, origin: URL): boolean;
63
+ /**
64
+ * Check whether `target` may be enqueued under `strategy` relative to `origin`: it must use an `http(s)`
65
+ * scheme and match the strategy. On rejection, `reason` is a human-readable message for log output.
66
+ */
67
+ export declare function filterUrl(target: string | URL, origin: string | URL, strategy: EnqueueStrategy | `${EnqueueStrategy}`): {
68
+ allowed: boolean;
69
+ reason?: string;
70
+ };
2
71
  /**
3
72
  * Appends search (query string) parameters to a URL, replacing the original value (if any).
4
73
  *
@@ -7,4 +76,3 @@ import type { SearchParams } from '@crawlee/types';
7
76
  * @internal
8
77
  */
9
78
  export declare function applySearchParams(url: URL, searchParams: SearchParams | undefined): void;
10
- //# sourceMappingURL=url.d.ts.map