@crawlee/utils 4.0.0-beta.26 → 4.0.0-beta.260

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +10 -17
  3. package/index.js +8 -16
  4. package/internal.d.ts +12 -0
  5. package/internal.js +11 -0
  6. package/internals/blocked.d.ts +0 -1
  7. package/internals/blocked.js +0 -1
  8. package/internals/cheerio.d.ts +1 -3
  9. package/internals/cheerio.js +25 -16
  10. package/internals/extract-microdata.d.ts +24 -0
  11. package/internals/extract-microdata.js +118 -0
  12. package/internals/extract-urls.d.ts +1 -2
  13. package/internals/extract-urls.js +18 -17
  14. package/internals/general.d.ts +0 -25
  15. package/internals/general.js +2 -110
  16. package/internals/http-request.d.ts +5 -0
  17. package/internals/http-request.js +36 -0
  18. package/internals/iterables.d.ts +0 -80
  19. package/internals/iterables.js +0 -126
  20. package/internals/mutation-journal.d.ts +29 -0
  21. package/internals/mutation-journal.js +173 -0
  22. package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
  23. package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
  24. package/internals/robots.d.ts +35 -15
  25. package/internals/robots.js +66 -30
  26. package/internals/schemas.d.ts +114 -0
  27. package/internals/schemas.js +115 -0
  28. package/internals/sitemap.d.ts +44 -3
  29. package/internals/sitemap.js +173 -97
  30. package/internals/social.d.ts +4 -3
  31. package/internals/social.js +36 -11
  32. package/internals/url.d.ts +67 -1
  33. package/internals/url.js +118 -2
  34. package/internals/validation.d.ts +9 -0
  35. package/internals/validation.js +16 -0
  36. package/package.json +16 -15
  37. package/index.d.ts.map +0 -1
  38. package/index.js.map +0 -1
  39. package/internals/blocked.d.ts.map +0 -1
  40. package/internals/blocked.js.map +0 -1
  41. package/internals/cheerio.d.ts.map +0 -1
  42. package/internals/cheerio.js.map +0 -1
  43. package/internals/chunk.d.ts +0 -2
  44. package/internals/chunk.d.ts.map +0 -1
  45. package/internals/chunk.js +0 -40
  46. package/internals/chunk.js.map +0 -1
  47. package/internals/debug.d.ts +0 -32
  48. package/internals/debug.d.ts.map +0 -1
  49. package/internals/debug.js +0 -41
  50. package/internals/debug.js.map +0 -1
  51. package/internals/extract-urls.d.ts.map +0 -1
  52. package/internals/extract-urls.js.map +0 -1
  53. package/internals/general.d.ts.map +0 -1
  54. package/internals/general.js.map +0 -1
  55. package/internals/iterables.d.ts.map +0 -1
  56. package/internals/iterables.js.map +0 -1
  57. package/internals/open_graph_parser.d.ts.map +0 -1
  58. package/internals/open_graph_parser.js.map +0 -1
  59. package/internals/robots.d.ts.map +0 -1
  60. package/internals/robots.js.map +0 -1
  61. package/internals/sitemap.d.ts.map +0 -1
  62. package/internals/sitemap.js.map +0 -1
  63. package/internals/social.d.ts.map +0 -1
  64. package/internals/social.js.map +0 -1
  65. package/internals/system-info/cpu-info.d.ts +0 -64
  66. package/internals/system-info/cpu-info.d.ts.map +0 -1
  67. package/internals/system-info/cpu-info.js +0 -211
  68. package/internals/system-info/cpu-info.js.map +0 -1
  69. package/internals/system-info/memory-info.d.ts +0 -28
  70. package/internals/system-info/memory-info.d.ts.map +0 -1
  71. package/internals/system-info/memory-info.js +0 -118
  72. package/internals/system-info/memory-info.js.map +0 -1
  73. package/internals/system-info/ps-tree.d.ts +0 -18
  74. package/internals/system-info/ps-tree.d.ts.map +0 -1
  75. package/internals/system-info/ps-tree.js +0 -145
  76. package/internals/system-info/ps-tree.js.map +0 -1
  77. package/internals/typedefs.d.ts +0 -5
  78. package/internals/typedefs.d.ts.map +0 -1
  79. package/internals/typedefs.js +0 -9
  80. package/internals/typedefs.js.map +0 -1
  81. package/internals/url.d.ts.map +0 -1
  82. package/internals/url.js.map +0 -1
@@ -3,132 +3,134 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
3
3
  import { StringDecoder } from 'node:string_decoder';
4
4
  import { createGunzip } from 'node:zlib';
5
5
  import { FetchHttpClient } from '@crawlee/http-client';
6
- import { fileTypeStream } from 'file-type';
7
- import sax from 'sax';
8
- import MIMEType from 'whatwg-mimetype';
9
- import log from '@apify/log';
6
+ import { MIMEType } from 'whatwg-mimetype';
10
7
  import { mergeAsyncIterables } from './iterables.js';
11
- import { RobotsFile } from './robots.js';
8
+ import { RobotsTxtFile } from './robots.js';
9
+ import { filterUrl } from './url.js';
12
10
  class SitemapTxtParser extends Transform {
13
- decoder = new StringDecoder('utf8');
14
- buffer = '';
11
+ #decoder = new StringDecoder('utf8');
12
+ #buffer = '';
15
13
  constructor() {
16
14
  super({
17
15
  readableObjectMode: true,
18
16
  transform: (chunk, _encoding, callback) => {
19
- this.processBuffer(this.decoder.write(chunk), false);
17
+ this.#processBuffer(this.#decoder.write(chunk), false);
20
18
  callback();
21
19
  },
22
20
  flush: (callback) => {
23
- this.processBuffer(this.decoder.end(), true);
21
+ this.#processBuffer(this.#decoder.end(), true);
24
22
  callback();
25
23
  },
26
24
  });
27
25
  }
28
- processBuffer(input, finalize) {
29
- this.buffer += input;
30
- if (finalize || this.buffer.includes('\n')) {
31
- const parts = this.buffer
32
- .split('\n')
33
- .map((part) => part.trim())
34
- .filter((part) => part.length > 0);
35
- if (finalize) {
36
- for (const url of parts) {
37
- this.push({ type: 'url', loc: url });
38
- }
39
- this.buffer = '';
40
- }
41
- else if (parts.length > 0) {
42
- for (const url of parts.slice(0, -1)) {
43
- this.push({ type: 'url', loc: url });
44
- }
45
- this.buffer = parts.at(-1);
26
+ #processBuffer(input, finalize) {
27
+ const lines = (this.#buffer + input).split('\n');
28
+ // The text after the last line break may be a partial line, so it waits for the next chunk.
29
+ this.#buffer = finalize ? '' : lines.pop();
30
+ for (const line of lines) {
31
+ const url = line.trim();
32
+ if (url.length > 0) {
33
+ this.push({ type: 'url', loc: url });
46
34
  }
47
35
  }
48
36
  }
49
37
  }
50
38
  class SitemapXmlParser extends Transform {
51
- decoder = new StringDecoder('utf8');
52
- parser = new sax.SAXParser(true);
53
- rootTagName;
54
- currentTag = undefined;
55
- url = {};
56
- constructor() {
39
+ #decoder = new StringDecoder('utf8');
40
+ #parser;
41
+ #rootTagName;
42
+ #currentTag = undefined;
43
+ #url = {};
44
+ static async create() {
45
+ const { SAXParser } = (await import('sax')).default;
46
+ return new SitemapXmlParser(new SAXParser(true));
47
+ }
48
+ constructor(parser) {
57
49
  super({
58
50
  readableObjectMode: true,
59
51
  transform: (chunk, _encoding, callback) => {
60
- this.parser.write(this.decoder.write(chunk));
52
+ this.#parser.write(this.#decoder.write(chunk));
61
53
  callback();
62
54
  },
63
55
  flush: (callback) => {
64
- const rest = this.decoder.end();
56
+ const rest = this.#decoder.end();
65
57
  if (rest.length > 0) {
66
- this.parser.write(rest);
58
+ this.#parser.write(rest);
67
59
  }
68
- this.parser.end();
60
+ this.#parser.end();
69
61
  callback();
70
62
  },
71
63
  });
72
- this.parser.onopentag = this.onOpenTag.bind(this);
73
- this.parser.onclosetag = this.onCloseTag.bind(this);
74
- this.parser.ontext = this.onText.bind(this);
75
- this.parser.oncdata = this.onText.bind(this);
76
- this.parser.onerror = this.destroy.bind(this);
64
+ this.#parser = parser;
65
+ this.#parser.onopentag = this.#onOpenTag.bind(this);
66
+ this.#parser.onclosetag = this.#onCloseTag.bind(this);
67
+ this.#parser.ontext = this.#onText.bind(this);
68
+ this.#parser.oncdata = this.#onText.bind(this);
69
+ this.#parser.onerror = this.destroy.bind(this);
77
70
  }
78
- onOpenTag(node) {
79
- if (this.rootTagName !== undefined) {
71
+ #onOpenTag(node) {
72
+ if (this.#rootTagName !== undefined) {
80
73
  if (node.name === 'loc' ||
81
74
  node.name === 'lastmod' ||
82
75
  node.name === 'priority' ||
83
76
  node.name === 'changefreq') {
84
- this.currentTag = node.name;
77
+ this.#currentTag = node.name;
85
78
  }
86
79
  }
87
80
  if (node.name === 'urlset') {
88
- this.rootTagName = 'urlset';
81
+ this.#rootTagName = 'urlset';
89
82
  }
90
83
  if (node.name === 'sitemapindex') {
91
- this.rootTagName = 'sitemapindex';
84
+ this.#rootTagName = 'sitemapindex';
92
85
  }
93
86
  }
94
- onCloseTag(name) {
87
+ #onCloseTag(name) {
95
88
  if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
96
- this.currentTag = undefined;
89
+ this.#currentTag = undefined;
97
90
  }
98
- if (name === 'url' && this.url.loc !== undefined) {
99
- this.push({ type: 'url', ...this.url, loc: this.url.loc });
100
- this.url = {};
91
+ if (name === 'url') {
92
+ if (this.#url.loc !== undefined) {
93
+ this.push({ type: 'url', ...this.#url, loc: this.#url.loc });
94
+ }
95
+ this.#url = {};
101
96
  }
102
97
  }
103
- onText(text) {
104
- if (this.currentTag === 'loc') {
105
- if (this.rootTagName === 'sitemapindex') {
106
- this.push({ type: 'sitemapUrl', url: text.trim() });
98
+ #onText(text) {
99
+ text = text.trim();
100
+ // Whitespace around a CDATA section arrives as separate text events.
101
+ if (text === '') {
102
+ return;
103
+ }
104
+ if (this.#currentTag === 'loc') {
105
+ if (this.#rootTagName === 'sitemapindex') {
106
+ this.push({ type: 'sitemapUrl', url: text });
107
107
  }
108
- if (this.rootTagName === 'urlset') {
109
- this.url ??= {};
110
- this.url.loc = text.trim();
108
+ if (this.#rootTagName === 'urlset') {
109
+ this.#url ??= {};
110
+ this.#url.loc = text;
111
111
  }
112
112
  }
113
- text = text.trim();
114
- if (this.currentTag === 'lastmod') {
115
- this.url.lastmod = new Date(text);
113
+ if (this.#currentTag === 'lastmod') {
114
+ const lastmod = new Date(text);
115
+ if (!Number.isNaN(lastmod.getTime())) {
116
+ this.#url.lastmod = lastmod;
117
+ }
116
118
  }
117
- if (this.currentTag === 'priority') {
118
- this.url.priority = Number(text);
119
+ if (this.#currentTag === 'priority') {
120
+ this.#url.priority = Number(text);
119
121
  }
120
- if (this.currentTag === 'changefreq') {
122
+ if (this.#currentTag === 'changefreq') {
121
123
  if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
122
- this.url.changefreq = text;
124
+ this.#url.changefreq = text;
123
125
  }
124
126
  }
125
127
  }
126
128
  }
127
129
  export async function* parseSitemap(initialSources, proxyUrl, options) {
128
- const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, } = options ?? {};
130
+ const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, enqueueStrategy = 'same-hostname', logger, } = options ?? {};
129
131
  const sources = [...initialSources];
130
132
  const visitedSitemapUrls = new Set();
131
- const createParser = (contentType = '', url) => {
133
+ const createParser = async (contentType = '', url) => {
132
134
  let mimeType;
133
135
  try {
134
136
  mimeType = new MIMEType(contentType);
@@ -137,7 +139,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
137
139
  mimeType = null;
138
140
  }
139
141
  if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
140
- return new SitemapXmlParser();
142
+ return SitemapXmlParser.create();
141
143
  }
142
144
  if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
143
145
  return new SitemapTxtParser();
@@ -147,12 +149,13 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
147
149
  while (sources.length > 0) {
148
150
  const source = sources.shift();
149
151
  if ((source?.depth ?? 0) > maxDepth) {
150
- log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
151
152
  continue;
152
153
  }
153
154
  let items = null;
155
+ // Parent URL, parsed once and reused as the origin for the strategy checks below.
156
+ let sitemapUrl;
154
157
  if (source.type === 'url') {
155
- const sitemapUrl = new URL(source.url);
158
+ sitemapUrl = new URL(source.url);
156
159
  visitedSitemapUrls.add(sitemapUrl.toString());
157
160
  let retriesLeft = sitemapRetries + 1;
158
161
  while (retriesLeft-- > 0) {
@@ -162,11 +165,11 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
162
165
  sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
163
166
  method: 'GET',
164
167
  headers: {
165
- accept: 'text/plain, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8',
168
+ accept: '*/*',
166
169
  },
167
170
  }), {
168
171
  proxyUrl,
169
- timeout,
172
+ timeoutMillis: timeout,
170
173
  });
171
174
  }
172
175
  catch (error) {
@@ -178,6 +181,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
178
181
  if (sitemapResponse.body === null) {
179
182
  break;
180
183
  }
184
+ const { fileTypeStream } = await import('file-type');
181
185
  const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
182
186
  if (streamWithType.fileType !== undefined) {
183
187
  contentType = streamWithType.fileType.mime;
@@ -191,7 +195,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
191
195
  sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
192
196
  }
193
197
  }
194
- items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
198
+ items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), await createParser(contentType ?? undefined, sitemapUrl), (e) => {
195
199
  if (e !== undefined && e !== null) {
196
200
  error = { type: 'parser', error: e };
197
201
  }
@@ -214,28 +218,53 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
214
218
  }
215
219
  }
216
220
  catch (e) {
217
- log.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
221
+ logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
218
222
  }
219
223
  }
220
224
  }
221
225
  else if (source.type === 'raw') {
222
- items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
226
+ items = pipeline(Readable.from([source.content]), await createParser('text/xml'), (error) => {
223
227
  if (error !== undefined) {
224
- log.warning(`Malformed sitemap content: ${error}`);
228
+ logger?.warning(`Malformed sitemap content: ${error}`);
225
229
  }
226
230
  });
227
231
  }
228
232
  if (items === null) {
229
233
  continue;
230
234
  }
235
+ // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
236
+ // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
237
+ let droppedUrlEntries = 0;
231
238
  for await (const item of items) {
232
239
  if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
240
+ if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
241
+ logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
242
+ continue;
243
+ }
244
+ // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
245
+ // parent. Raw string sources have no parent URL, so the check is skipped.
246
+ if (source.type === 'url') {
247
+ const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
248
+ if (!allowed) {
249
+ logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
250
+ continue;
251
+ }
252
+ }
233
253
  sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
234
254
  if (emitNestedSitemaps) {
235
255
  yield { loc: item.url, originSitemapUrl: null };
236
256
  }
237
257
  }
238
258
  if (item.type === 'url') {
259
+ // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
260
+ if (source.type === 'url') {
261
+ const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
262
+ if (!allowed) {
263
+ droppedUrlEntries++;
264
+ logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
265
+ continue;
266
+ }
267
+ }
239
268
  yield {
240
269
  ...item,
241
270
  originSitemapUrl: source.type === 'url'
@@ -244,6 +273,9 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
244
273
  };
245
274
  }
246
275
  }
276
+ if (droppedUrlEntries > 0 && source.type === 'url') {
277
+ logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
278
+ }
247
279
  }
248
280
  }
249
281
  /**
@@ -285,7 +317,7 @@ export class Sitemap {
285
317
  * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
286
318
  */
287
319
  static async load(urls, proxyUrl, parseSitemapOptions) {
288
- return await this.parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
320
+ return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
289
321
  }
290
322
  /**
291
323
  * Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
@@ -293,16 +325,17 @@ export class Sitemap {
293
325
  * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
294
326
  */
295
327
  static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
296
- return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
328
+ return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
297
329
  }
298
- static async parse(sources, proxyUrl, parseSitemapOptions) {
330
+ static async #parse(sources, proxyUrl, parseSitemapOptions) {
299
331
  const urls = [];
300
332
  try {
301
333
  for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
302
334
  urls.push(item.loc);
303
335
  }
304
336
  }
305
- catch {
337
+ catch (e) {
338
+ parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
306
339
  return new Sitemap([]);
307
340
  }
308
341
  return new Sitemap(urls);
@@ -316,7 +349,19 @@ export class Sitemap {
316
349
  * @returns An async iterable with the discovered sitemap URLs.
317
350
  */
318
351
  export async function* discoverValidSitemaps(urls, options = {}) {
319
- const { proxyUrl, httpClient = new FetchHttpClient() } = options;
352
+ const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
353
+ const controller = new AbortController();
354
+ const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
355
+ const onExternalAbort = () => controller.abort();
356
+ if (externalSignal) {
357
+ if (externalSignal.aborted) {
358
+ controller.abort();
359
+ }
360
+ else {
361
+ externalSignal.addEventListener('abort', onExternalAbort, { once: true });
362
+ }
363
+ }
364
+ const signal = controller.signal;
320
365
  const sitemapUrls = new Set();
321
366
  const addSitemapUrl = (url) => {
322
367
  const sizeBefore = sitemapUrls.size;
@@ -331,7 +376,11 @@ export async function* discoverValidSitemaps(urls, options = {}) {
331
376
  return false;
332
377
  }
333
378
  try {
334
- const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), { proxyUrl });
379
+ const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
380
+ proxyUrl,
381
+ timeoutMillis: requestTimeoutMillis,
382
+ signal,
383
+ });
335
384
  return response.status >= 200 && response.status < 400;
336
385
  }
337
386
  catch {
@@ -343,17 +392,24 @@ export async function* discoverValidSitemaps(urls, options = {}) {
343
392
  return;
344
393
  }
345
394
  try {
346
- const robotsFile = await RobotsFile.find(domainUrls[0], { proxyUrl, httpClient });
347
- for (const sitemapUrl of robotsFile.getSitemaps()) {
395
+ const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
396
+ proxyUrl,
397
+ timeoutMillis: requestTimeoutMillis,
398
+ signal,
399
+ httpClient,
400
+ logger,
401
+ });
402
+ // Surface all referenced sitemaps, including cross-host; scoping happens at load time.
403
+ for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
348
404
  if (addSitemapUrl(sitemapUrl)) {
349
405
  yield sitemapUrl;
350
406
  }
351
407
  }
352
408
  }
353
409
  catch (err) {
354
- log.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
410
+ logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
355
411
  }
356
- const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
412
+ const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));
357
413
  if (sitemapUrl !== undefined) {
358
414
  if (addSitemapUrl(sitemapUrl)) {
359
415
  yield sitemapUrl;
@@ -361,14 +417,29 @@ export async function* discoverValidSitemaps(urls, options = {}) {
361
417
  }
362
418
  else {
363
419
  const firstUrl = new URL(domainUrls[0]);
420
+ // The guessed candidates are well-known paths on the same origin, so the query string
421
+ // and fragment of the input URL are not part of them - carrying them over probes (and
422
+ // yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
423
+ firstUrl.search = '';
424
+ firstUrl.hash = '';
364
425
  const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
365
- for (const pathname of possibleSitemapPathnames) {
426
+ const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
366
427
  firstUrl.pathname = pathname;
367
- if (await urlExists(firstUrl.toString())) {
368
- if (addSitemapUrl(firstUrl.toString())) {
369
- yield firstUrl.toString();
428
+ return firstUrl.toString();
429
+ });
430
+ const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
431
+ for (const [index, result] of candidateResults.entries()) {
432
+ const candidateSitemapUrl = candidateSitemapUrls[index];
433
+ if (result.status === 'fulfilled') {
434
+ if (result.value && addSitemapUrl(candidateSitemapUrl)) {
435
+ yield candidateSitemapUrl;
370
436
  }
371
437
  }
438
+ else {
439
+ logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
440
+ error: result.reason,
441
+ });
442
+ }
372
443
  }
373
444
  }
374
445
  };
@@ -380,12 +451,17 @@ export async function* discoverValidSitemaps(urls, options = {}) {
380
451
  }, {});
381
452
  const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
382
453
  const discoveredUrls = new Set();
383
- for await (const url of mergeAsyncIterables(...iterables)) {
384
- if (discoveredUrls.has(url)) {
385
- continue;
454
+ try {
455
+ for await (const url of mergeAsyncIterables(...iterables)) {
456
+ if (discoveredUrls.has(url)) {
457
+ continue;
458
+ }
459
+ discoveredUrls.add(url);
460
+ yield url;
386
461
  }
387
- discoveredUrls.add(url);
388
- yield url;
462
+ }
463
+ finally {
464
+ clearTimeout(timeoutHandle);
465
+ externalSignal?.removeEventListener('abort', onExternalAbort);
389
466
  }
390
467
  }
391
- //# sourceMappingURL=sitemap.js.map
@@ -19,6 +19,8 @@ export declare function emailsFromText(text: string): string[];
19
19
  /**
20
20
  * The function extracts email addresses from a list of URLs.
21
21
  * Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
22
+ * A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
23
+ * such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
22
24
  * Note that the function preserves the order of emails and keep duplicates.
23
25
  * @param urls Array of URLs.
24
26
  * @return Array of emails addresses found.
@@ -225,7 +227,7 @@ export declare const TWITTER_REGEX: RegExp;
225
227
  * ```
226
228
  * import { social } from 'crawlee';
227
229
  *
228
- * const matches = text.match(social.TWITTER_REGEX_STRING);
230
+ * const matches = text.match(social.TWITTER_REGEX_GLOBAL);
229
231
  * if (matches) console.log(`${matches.length} Twitter profiles found!`);
230
232
  * ```
231
233
  */
@@ -488,5 +490,4 @@ export declare const DISCORD_REGEX_GLOBAL: RegExp;
488
490
  * so that the caller doesn't need to parse the HTML document again, if needed.
489
491
  * @return An object with the social handles.
490
492
  */
491
- export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles;
492
- //# sourceMappingURL=social.d.ts.map
493
+ export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;
@@ -1,7 +1,9 @@
1
- import * as cheerio from 'cheerio';
2
1
  import { htmlToText } from './cheerio.js';
3
2
  // Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
4
- const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
3
+ // The dot-atom local part and domain labels use RFC 5321 length bounds ({1,64}, {0,62})
4
+ // instead of unbounded quantifiers to avoid quadratic backtracking (ReDoS) on long
5
+ // dotted or hyphenated inputs, e.g. text scraped by parseHandlesFromHtml().
6
+ const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}){0,32}|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
5
7
  /**
6
8
  * Regular expression to exactly match a single email address.
7
9
  * It has the following form: `/^...$/i`.
@@ -28,6 +30,8 @@ export function emailsFromText(text) {
28
30
  /**
29
31
  * The function extracts email addresses from a list of URLs.
30
32
  * Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
33
+ * A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
34
+ * such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
31
35
  * Note that the function preserves the order of emails and keep duplicates.
32
36
  * @param urls Array of URLs.
33
37
  * @return Array of emails addresses found.
@@ -42,9 +46,20 @@ export function emailsFromUrls(urls) {
42
46
  continue;
43
47
  if (!EMAIL_URL_PREFIX_REGEX.test(url))
44
48
  continue;
45
- const email = url.replace(EMAIL_URL_PREFIX_REGEX, '').trim();
46
- if (EMAIL_REGEX.test(email))
47
- emails.push(email);
49
+ // Characters like `?` and `,` inside an address must be percent-encoded, so it's safe
50
+ // to split on them before decoding.
51
+ const [addresses] = url.replace(EMAIL_URL_PREFIX_REGEX, '').split('?');
52
+ for (const address of addresses.split(',')) {
53
+ let email = address.trim();
54
+ try {
55
+ email = decodeURIComponent(email).trim();
56
+ }
57
+ catch {
58
+ // Keep the raw value if it isn't valid percent-encoding
59
+ }
60
+ if (EMAIL_REGEX.test(email))
61
+ emails.push(email);
62
+ }
48
63
  }
49
64
  return emails;
50
65
  }
@@ -136,8 +151,18 @@ export function phonesFromUrls(urls) {
136
151
  continue;
137
152
  if (!PHONE_URL_PREFIX_REGEX.test(url))
138
153
  continue;
139
- const phone = url.replace(PHONE_URL_PREFIX_REGEX, '').trim();
140
- if (PHONE_REGEX.test(phone))
154
+ // Per RFC 3966, parameters such as `;ext=89` can follow the number, and `-`, `.`, `(` and `)`
155
+ // are only visual separators. They are checked without separators too, so a one-digit country
156
+ // code like in `tel:+1-555-123-4567` is accepted.
157
+ const [number] = url.replace(PHONE_URL_PREFIX_REGEX, '').split(';');
158
+ let phone = number.trim();
159
+ try {
160
+ phone = decodeURIComponent(phone).trim();
161
+ }
162
+ catch {
163
+ // Keep the raw value if it isn't valid percent-encoding
164
+ }
165
+ if (PHONE_REGEX.test(phone) || PHONE_REGEX.test(phone.replace(/[-.() ]/g, '')))
141
166
  phones.push(phone);
142
167
  }
143
168
  return phones;
@@ -324,7 +349,7 @@ export const TWITTER_REGEX = new RegExp(`^${TWITTER_REGEX_STRING}$`, 'i');
324
349
  * ```
325
350
  * import { social } from 'crawlee';
326
351
  *
327
- * const matches = text.match(social.TWITTER_REGEX_STRING);
352
+ * const matches = text.match(social.TWITTER_REGEX_GLOBAL);
328
353
  * if (matches) console.log(`${matches.length} Twitter profiles found!`);
329
354
  * ```
330
355
  */
@@ -587,7 +612,8 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
587
612
  * so that the caller doesn't need to parse the HTML document again, if needed.
588
613
  * @return An object with the social handles.
589
614
  */
590
- export function parseHandlesFromHtml(html, data = null) {
615
+ export async function parseHandlesFromHtml(html, data = null) {
616
+ const cheerio = await import('cheerio/slim');
591
617
  const result = {
592
618
  emails: [],
593
619
  phones: [],
@@ -606,7 +632,7 @@ export function parseHandlesFromHtml(html, data = null) {
606
632
  const $ = cheerio.load(html, { xml: { decodeEntities: true } });
607
633
  if (data)
608
634
  data.$ = $;
609
- const text = htmlToText($);
635
+ const text = await htmlToText($);
610
636
  if (data)
611
637
  data.text = text;
612
638
  // NOTE: we need to parse each text separately, orherwise we might concatenate unrelated texts
@@ -644,4 +670,3 @@ export function parseHandlesFromHtml(html, data = null) {
644
670
  }
645
671
  return result;
646
672
  }
647
- //# sourceMappingURL=social.js.map