@crawlee/utils 4.0.0-beta.29 → 4.0.0-beta.290

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +10 -17
  3. package/index.js +8 -16
  4. package/internal.d.ts +12 -0
  5. package/internal.js +11 -0
  6. package/internals/blocked.d.ts +7 -1
  7. package/internals/blocked.js +11 -1
  8. package/internals/cheerio.d.ts +1 -3
  9. package/internals/cheerio.js +33 -23
  10. package/internals/extract-microdata.d.ts +24 -0
  11. package/internals/extract-microdata.js +136 -0
  12. package/internals/extract-urls.d.ts +1 -2
  13. package/internals/extract-urls.js +18 -23
  14. package/internals/general.d.ts +0 -25
  15. package/internals/general.js +2 -110
  16. package/internals/http-request.d.ts +5 -0
  17. package/internals/http-request.js +36 -0
  18. package/internals/iterables.d.ts +0 -80
  19. package/internals/iterables.js +0 -126
  20. package/internals/mutation-journal.d.ts +29 -0
  21. package/internals/mutation-journal.js +173 -0
  22. package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
  23. package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
  24. package/internals/robots.d.ts +35 -15
  25. package/internals/robots.js +66 -30
  26. package/internals/schemas.d.ts +114 -0
  27. package/internals/schemas.js +115 -0
  28. package/internals/sitemap.d.ts +44 -3
  29. package/internals/sitemap.js +243 -116
  30. package/internals/social.d.ts +4 -3
  31. package/internals/social.js +36 -11
  32. package/internals/url.d.ts +67 -1
  33. package/internals/url.js +118 -2
  34. package/internals/validation.d.ts +9 -0
  35. package/internals/validation.js +16 -0
  36. package/package.json +16 -15
  37. package/index.d.ts.map +0 -1
  38. package/index.js.map +0 -1
  39. package/internals/blocked.d.ts.map +0 -1
  40. package/internals/blocked.js.map +0 -1
  41. package/internals/cheerio.d.ts.map +0 -1
  42. package/internals/cheerio.js.map +0 -1
  43. package/internals/chunk.d.ts +0 -2
  44. package/internals/chunk.d.ts.map +0 -1
  45. package/internals/chunk.js +0 -40
  46. package/internals/chunk.js.map +0 -1
  47. package/internals/debug.d.ts +0 -32
  48. package/internals/debug.d.ts.map +0 -1
  49. package/internals/debug.js +0 -41
  50. package/internals/debug.js.map +0 -1
  51. package/internals/extract-urls.d.ts.map +0 -1
  52. package/internals/extract-urls.js.map +0 -1
  53. package/internals/general.d.ts.map +0 -1
  54. package/internals/general.js.map +0 -1
  55. package/internals/iterables.d.ts.map +0 -1
  56. package/internals/iterables.js.map +0 -1
  57. package/internals/open_graph_parser.d.ts.map +0 -1
  58. package/internals/open_graph_parser.js.map +0 -1
  59. package/internals/robots.d.ts.map +0 -1
  60. package/internals/robots.js.map +0 -1
  61. package/internals/sitemap.d.ts.map +0 -1
  62. package/internals/sitemap.js.map +0 -1
  63. package/internals/social.d.ts.map +0 -1
  64. package/internals/social.js.map +0 -1
  65. package/internals/system-info/cpu-info.d.ts +0 -64
  66. package/internals/system-info/cpu-info.d.ts.map +0 -1
  67. package/internals/system-info/cpu-info.js +0 -211
  68. package/internals/system-info/cpu-info.js.map +0 -1
  69. package/internals/system-info/memory-info.d.ts +0 -28
  70. package/internals/system-info/memory-info.d.ts.map +0 -1
  71. package/internals/system-info/memory-info.js +0 -118
  72. package/internals/system-info/memory-info.js.map +0 -1
  73. package/internals/system-info/ps-tree.d.ts +0 -18
  74. package/internals/system-info/ps-tree.d.ts.map +0 -1
  75. package/internals/system-info/ps-tree.js +0 -145
  76. package/internals/system-info/ps-tree.js.map +0 -1
  77. package/internals/typedefs.d.ts +0 -5
  78. package/internals/typedefs.d.ts.map +0 -1
  79. package/internals/typedefs.js +0 -9
  80. package/internals/typedefs.js.map +0 -1
  81. package/internals/url.d.ts.map +0 -1
  82. package/internals/url.js.map +0 -1
@@ -3,132 +3,157 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
3
3
  import { StringDecoder } from 'node:string_decoder';
4
4
  import { createGunzip } from 'node:zlib';
5
5
  import { FetchHttpClient } from '@crawlee/http-client';
6
- import { fileTypeStream } from 'file-type';
7
- import sax from 'sax';
8
- import MIMEType from 'whatwg-mimetype';
9
- import log from '@apify/log';
6
+ import { MIMEType } from 'whatwg-mimetype';
10
7
  import { mergeAsyncIterables } from './iterables.js';
11
- import { RobotsFile } from './robots.js';
8
+ import { RobotsTxtFile } from './robots.js';
9
+ import { filterUrl } from './url.js';
12
10
  class SitemapTxtParser extends Transform {
13
- decoder = new StringDecoder('utf8');
14
- buffer = '';
11
+ #decoder = new StringDecoder('utf8');
12
+ #buffer = '';
15
13
  constructor() {
16
14
  super({
17
15
  readableObjectMode: true,
18
16
  transform: (chunk, _encoding, callback) => {
19
- this.processBuffer(this.decoder.write(chunk), false);
17
+ this.#processBuffer(this.#decoder.write(chunk), false);
20
18
  callback();
21
19
  },
22
20
  flush: (callback) => {
23
- this.processBuffer(this.decoder.end(), true);
21
+ this.#processBuffer(this.#decoder.end(), true);
24
22
  callback();
25
23
  },
26
24
  });
27
25
  }
28
- processBuffer(input, finalize) {
29
- this.buffer += input;
30
- if (finalize || this.buffer.includes('\n')) {
31
- const parts = this.buffer
32
- .split('\n')
33
- .map((part) => part.trim())
34
- .filter((part) => part.length > 0);
35
- if (finalize) {
36
- for (const url of parts) {
37
- this.push({ type: 'url', loc: url });
38
- }
39
- this.buffer = '';
40
- }
41
- else if (parts.length > 0) {
42
- for (const url of parts.slice(0, -1)) {
43
- this.push({ type: 'url', loc: url });
44
- }
45
- this.buffer = parts.at(-1);
26
+ #processBuffer(input, finalize) {
27
+ const lines = (this.#buffer + input).split('\n');
28
+ // The text after the last line break may be a partial line, so it waits for the next chunk.
29
+ this.#buffer = finalize ? '' : lines.pop();
30
+ for (const line of lines) {
31
+ const url = line.trim();
32
+ if (url.length > 0) {
33
+ this.push({ type: 'url', loc: url });
46
34
  }
47
35
  }
48
36
  }
49
37
  }
38
+ /** Namespaces of the sitemap protocol (https://www.sitemaps.org/protocol.html), including Google's legacy ones. */
39
+ const SITEMAP_NAMESPACES = new Set([
40
+ 'http://www.sitemaps.org/schemas/sitemap/0.9',
41
+ 'https://www.sitemaps.org/schemas/sitemap/0.9',
42
+ 'http://www.google.com/schemas/sitemap/0.9',
43
+ 'http://www.google.com/schemas/sitemap/0.84',
44
+ ]);
50
45
  class SitemapXmlParser extends Transform {
51
- decoder = new StringDecoder('utf8');
52
- parser = new sax.SAXParser(true);
53
- rootTagName;
54
- currentTag = undefined;
55
- url = {};
56
- constructor() {
46
+ #decoder = new StringDecoder('utf8');
47
+ #parser;
48
+ #rootTagName;
49
+ #currentTag = undefined;
50
+ #url = {};
51
+ /** Prefixes bound to the sitemap namespace, e.g. `ns0` for `<ns0:urlset xmlns:ns0="...">`. */
52
+ #sitemapPrefixes = new Set();
53
+ static async create() {
54
+ const { SAXParser } = (await import('sax')).default;
55
+ return new SitemapXmlParser(new SAXParser(true));
56
+ }
57
+ constructor(parser) {
57
58
  super({
58
59
  readableObjectMode: true,
59
60
  transform: (chunk, _encoding, callback) => {
60
- this.parser.write(this.decoder.write(chunk));
61
+ this.#parser.write(this.#decoder.write(chunk));
61
62
  callback();
62
63
  },
63
64
  flush: (callback) => {
64
- const rest = this.decoder.end();
65
+ const rest = this.#decoder.end();
65
66
  if (rest.length > 0) {
66
- this.parser.write(rest);
67
+ this.#parser.write(rest);
67
68
  }
68
- this.parser.end();
69
+ this.#parser.end();
69
70
  callback();
70
71
  },
71
72
  });
72
- this.parser.onopentag = this.onOpenTag.bind(this);
73
- this.parser.onclosetag = this.onCloseTag.bind(this);
74
- this.parser.ontext = this.onText.bind(this);
75
- this.parser.oncdata = this.onText.bind(this);
76
- this.parser.onerror = this.destroy.bind(this);
73
+ this.#parser = parser;
74
+ this.#parser.onopentag = this.#onOpenTag.bind(this);
75
+ this.#parser.onclosetag = this.#onCloseTag.bind(this);
76
+ this.#parser.ontext = this.#onText.bind(this);
77
+ this.#parser.oncdata = this.#onText.bind(this);
78
+ this.#parser.onerror = this.destroy.bind(this);
79
+ }
80
+ /**
81
+ * Returns the local name of a sitemap protocol element, or `undefined` for an element of another
82
+ * namespace (such as `image:loc`). Unprefixed elements are sitemap elements, as before.
83
+ */
84
+ #sitemapTagName(qualifiedName) {
85
+ const colon = qualifiedName.indexOf(':');
86
+ if (colon === -1)
87
+ return qualifiedName;
88
+ return this.#sitemapPrefixes.has(qualifiedName.slice(0, colon)) ? qualifiedName.slice(colon + 1) : undefined;
77
89
  }
78
- onOpenTag(node) {
79
- if (this.rootTagName !== undefined) {
80
- if (node.name === 'loc' ||
81
- node.name === 'lastmod' ||
82
- node.name === 'priority' ||
83
- node.name === 'changefreq') {
84
- this.currentTag = node.name;
90
+ #onOpenTag(node) {
91
+ for (const [attribute, value] of Object.entries(node.attributes)) {
92
+ if (attribute.startsWith('xmlns:') && SITEMAP_NAMESPACES.has(String(value).trim())) {
93
+ this.#sitemapPrefixes.add(attribute.slice('xmlns:'.length));
94
+ }
95
+ }
96
+ const name = this.#sitemapTagName(node.name);
97
+ if (this.#rootTagName !== undefined) {
98
+ if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
99
+ this.#currentTag = name;
85
100
  }
86
101
  }
87
- if (node.name === 'urlset') {
88
- this.rootTagName = 'urlset';
102
+ if (name === 'urlset') {
103
+ this.#rootTagName = 'urlset';
89
104
  }
90
- if (node.name === 'sitemapindex') {
91
- this.rootTagName = 'sitemapindex';
105
+ if (name === 'sitemapindex') {
106
+ this.#rootTagName = 'sitemapindex';
92
107
  }
93
108
  }
94
- onCloseTag(name) {
109
+ #onCloseTag(qualifiedName) {
110
+ const name = this.#sitemapTagName(qualifiedName);
95
111
  if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
96
- this.currentTag = undefined;
112
+ this.#currentTag = undefined;
97
113
  }
98
- if (name === 'url' && this.url.loc !== undefined) {
99
- this.push({ type: 'url', ...this.url, loc: this.url.loc });
100
- this.url = {};
114
+ if (name === 'url') {
115
+ if (this.#url.loc !== undefined) {
116
+ this.push({ type: 'url', ...this.#url, loc: this.#url.loc });
117
+ }
118
+ this.#url = {};
101
119
  }
102
120
  }
103
- onText(text) {
104
- if (this.currentTag === 'loc') {
105
- if (this.rootTagName === 'sitemapindex') {
106
- this.push({ type: 'sitemapUrl', url: text.trim() });
121
+ #onText(text) {
122
+ text = text.trim();
123
+ // Whitespace around a CDATA section arrives as separate text events.
124
+ if (text === '') {
125
+ return;
126
+ }
127
+ if (this.#currentTag === 'loc') {
128
+ if (this.#rootTagName === 'sitemapindex') {
129
+ this.push({ type: 'sitemapUrl', url: text });
107
130
  }
108
- if (this.rootTagName === 'urlset') {
109
- this.url ??= {};
110
- this.url.loc = text.trim();
131
+ if (this.#rootTagName === 'urlset') {
132
+ this.#url ??= {};
133
+ this.#url.loc = text;
111
134
  }
112
135
  }
113
- text = text.trim();
114
- if (this.currentTag === 'lastmod') {
115
- this.url.lastmod = new Date(text);
136
+ if (this.#currentTag === 'lastmod') {
137
+ const lastmod = new Date(text);
138
+ if (!Number.isNaN(lastmod.getTime())) {
139
+ this.#url.lastmod = lastmod;
140
+ }
116
141
  }
117
- if (this.currentTag === 'priority') {
118
- this.url.priority = Number(text);
142
+ if (this.#currentTag === 'priority') {
143
+ this.#url.priority = Number(text);
119
144
  }
120
- if (this.currentTag === 'changefreq') {
145
+ if (this.#currentTag === 'changefreq') {
121
146
  if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
122
- this.url.changefreq = text;
147
+ this.#url.changefreq = text;
123
148
  }
124
149
  }
125
150
  }
126
151
  }
127
152
  export async function* parseSitemap(initialSources, proxyUrl, options) {
128
- const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, } = options ?? {};
153
+ const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, enqueueStrategy = 'same-hostname', logger, } = options ?? {};
129
154
  const sources = [...initialSources];
130
155
  const visitedSitemapUrls = new Set();
131
- const createParser = (contentType = '', url) => {
156
+ const createParser = async (contentType = '', url) => {
132
157
  let mimeType;
133
158
  try {
134
159
  mimeType = new MIMEType(contentType);
@@ -137,22 +162,27 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
137
162
  mimeType = null;
138
163
  }
139
164
  if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
140
- return new SitemapXmlParser();
165
+ return SitemapXmlParser.create();
141
166
  }
142
167
  if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
143
168
  return new SitemapTxtParser();
144
169
  }
170
+ // A name like `sitemap.gz` says nothing about what was compressed. Sitemaps are XML unless named otherwise.
171
+ if (mimeType?.essence === 'application/gzip') {
172
+ return SitemapXmlParser.create();
173
+ }
145
174
  throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);
146
175
  };
147
176
  while (sources.length > 0) {
148
177
  const source = sources.shift();
149
178
  if ((source?.depth ?? 0) > maxDepth) {
150
- log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
151
179
  continue;
152
180
  }
153
181
  let items = null;
182
+ // Parent URL, parsed once and reused as the origin for the strategy checks below.
183
+ let sitemapUrl;
154
184
  if (source.type === 'url') {
155
- const sitemapUrl = new URL(source.url);
185
+ sitemapUrl = new URL(source.url);
156
186
  visitedSitemapUrls.add(sitemapUrl.toString());
157
187
  let retriesLeft = sitemapRetries + 1;
158
188
  while (retriesLeft-- > 0) {
@@ -162,11 +192,11 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
162
192
  sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
163
193
  method: 'GET',
164
194
  headers: {
165
- accept: 'text/plain, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8',
195
+ accept: '*/*',
166
196
  },
167
197
  }), {
168
198
  proxyUrl,
169
- timeout,
199
+ timeoutMillis: timeout,
170
200
  });
171
201
  }
172
202
  catch (error) {
@@ -178,6 +208,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
178
208
  if (sitemapResponse.body === null) {
179
209
  break;
180
210
  }
211
+ const { fileTypeStream } = await import('file-type');
181
212
  const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
182
213
  if (streamWithType.fileType !== undefined) {
183
214
  contentType = streamWithType.fileType.mime;
@@ -191,7 +222,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
191
222
  sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
192
223
  }
193
224
  }
194
- items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
225
+ items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), await createParser(contentType ?? undefined, sitemapUrl), (e) => {
195
226
  if (e !== undefined && e !== null) {
196
227
  error = { type: 'parser', error: e };
197
228
  }
@@ -214,35 +245,87 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
214
245
  }
215
246
  }
216
247
  catch (e) {
217
- log.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
248
+ logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
218
249
  }
219
250
  }
220
251
  }
221
252
  else if (source.type === 'raw') {
222
- items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
253
+ items = pipeline(Readable.from([source.content]), await createParser('text/xml'), (error) => {
223
254
  if (error !== undefined) {
224
- log.warning(`Malformed sitemap content: ${error}`);
255
+ logger?.warning(`Malformed sitemap content: ${error}`);
225
256
  }
226
257
  });
227
258
  }
228
259
  if (items === null) {
229
260
  continue;
230
261
  }
231
- for await (const item of items) {
232
- if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
233
- sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
234
- if (emitNestedSitemaps) {
235
- yield { loc: item.url, originSitemapUrl: null };
262
+ // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
263
+ // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
264
+ let droppedUrlEntries = 0;
265
+ // The same for every entry of this source, so compute it once.
266
+ const originSitemapUrl = source.type === 'url'
267
+ ? source.url
268
+ : `raw://${createHash('sha256').update(source.content).digest('base64')}`;
269
+ const iterator = items[Symbol.asyncIterator]();
270
+ try {
271
+ while (true) {
272
+ let next;
273
+ try {
274
+ next = await iterator.next();
275
+ }
276
+ catch (e) {
277
+ // Parsing failed: give up on this sitemap, keep the others.
278
+ // Raw sources already logged the error in their pipeline callback.
279
+ if (source.type !== 'raw') {
280
+ logger?.warning(`Malformed sitemap content: ${source.url} (${e})`);
281
+ }
282
+ break;
283
+ }
284
+ if (next.done)
285
+ break;
286
+ const item = next.value;
287
+ if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
288
+ if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
289
+ logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
290
+ continue;
291
+ }
292
+ // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
293
+ // parent. Raw string sources have no parent URL, so the check is skipped.
294
+ if (source.type === 'url') {
295
+ const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
296
+ if (!allowed) {
297
+ logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
298
+ continue;
299
+ }
300
+ }
301
+ sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
302
+ if (emitNestedSitemaps) {
303
+ yield { loc: item.url, originSitemapUrl: null };
304
+ }
305
+ }
306
+ if (item.type === 'url') {
307
+ // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
308
+ if (source.type === 'url') {
309
+ const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
310
+ if (!allowed) {
311
+ droppedUrlEntries++;
312
+ logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
313
+ continue;
314
+ }
315
+ }
316
+ yield {
317
+ ...item,
318
+ originSitemapUrl,
319
+ };
236
320
  }
237
321
  }
238
- if (item.type === 'url') {
239
- yield {
240
- ...item,
241
- originSitemapUrl: source.type === 'url'
242
- ? source.url
243
- : `raw://${createHash('sha256').update(source.content).digest('base64')}`,
244
- };
245
- }
322
+ }
323
+ finally {
324
+ // Mirrors `for await` cleanup: destroys the stream when we stop early or the consumer bails out.
325
+ await iterator.return?.();
326
+ }
327
+ if (droppedUrlEntries > 0 && source.type === 'url') {
328
+ logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
246
329
  }
247
330
  }
248
331
  }
@@ -285,7 +368,7 @@ export class Sitemap {
285
368
  * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
286
369
  */
287
370
  static async load(urls, proxyUrl, parseSitemapOptions) {
288
- return await this.parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
371
+ return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
289
372
  }
290
373
  /**
291
374
  * Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
@@ -293,16 +376,17 @@ export class Sitemap {
293
376
  * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
294
377
  */
295
378
  static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
296
- return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
379
+ return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
297
380
  }
298
- static async parse(sources, proxyUrl, parseSitemapOptions) {
381
+ static async #parse(sources, proxyUrl, parseSitemapOptions) {
299
382
  const urls = [];
300
383
  try {
301
384
  for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
302
385
  urls.push(item.loc);
303
386
  }
304
387
  }
305
- catch {
388
+ catch (e) {
389
+ parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
306
390
  return new Sitemap([]);
307
391
  }
308
392
  return new Sitemap(urls);
@@ -316,7 +400,19 @@ export class Sitemap {
316
400
  * @returns An async iterable with the discovered sitemap URLs.
317
401
  */
318
402
  export async function* discoverValidSitemaps(urls, options = {}) {
319
- const { proxyUrl, httpClient = new FetchHttpClient() } = options;
403
+ const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
404
+ const controller = new AbortController();
405
+ const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
406
+ const onExternalAbort = () => controller.abort();
407
+ if (externalSignal) {
408
+ if (externalSignal.aborted) {
409
+ controller.abort();
410
+ }
411
+ else {
412
+ externalSignal.addEventListener('abort', onExternalAbort, { once: true });
413
+ }
414
+ }
415
+ const signal = controller.signal;
320
416
  const sitemapUrls = new Set();
321
417
  const addSitemapUrl = (url) => {
322
418
  const sizeBefore = sitemapUrls.size;
@@ -331,7 +427,11 @@ export async function* discoverValidSitemaps(urls, options = {}) {
331
427
  return false;
332
428
  }
333
429
  try {
334
- const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), { proxyUrl });
430
+ const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
431
+ proxyUrl,
432
+ timeoutMillis: requestTimeoutMillis,
433
+ signal,
434
+ });
335
435
  return response.status >= 200 && response.status < 400;
336
436
  }
337
437
  catch {
@@ -343,17 +443,24 @@ export async function* discoverValidSitemaps(urls, options = {}) {
343
443
  return;
344
444
  }
345
445
  try {
346
- const robotsFile = await RobotsFile.find(domainUrls[0], { proxyUrl, httpClient });
347
- for (const sitemapUrl of robotsFile.getSitemaps()) {
446
+ const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
447
+ proxyUrl,
448
+ timeoutMillis: requestTimeoutMillis,
449
+ signal,
450
+ httpClient,
451
+ logger,
452
+ });
453
+ // Surface all referenced sitemaps, including cross-host; scoping happens at load time.
454
+ for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
348
455
  if (addSitemapUrl(sitemapUrl)) {
349
456
  yield sitemapUrl;
350
457
  }
351
458
  }
352
459
  }
353
460
  catch (err) {
354
- log.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
461
+ logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
355
462
  }
356
- const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
463
+ const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));
357
464
  if (sitemapUrl !== undefined) {
358
465
  if (addSitemapUrl(sitemapUrl)) {
359
466
  yield sitemapUrl;
@@ -361,14 +468,29 @@ export async function* discoverValidSitemaps(urls, options = {}) {
361
468
  }
362
469
  else {
363
470
  const firstUrl = new URL(domainUrls[0]);
471
+ // The guessed candidates are well-known paths on the same origin, so the query string
472
+ // and fragment of the input URL are not part of them - carrying them over probes (and
473
+ // yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
474
+ firstUrl.search = '';
475
+ firstUrl.hash = '';
364
476
  const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
365
- for (const pathname of possibleSitemapPathnames) {
477
+ const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
366
478
  firstUrl.pathname = pathname;
367
- if (await urlExists(firstUrl.toString())) {
368
- if (addSitemapUrl(firstUrl.toString())) {
369
- yield firstUrl.toString();
479
+ return firstUrl.toString();
480
+ });
481
+ const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
482
+ for (const [index, result] of candidateResults.entries()) {
483
+ const candidateSitemapUrl = candidateSitemapUrls[index];
484
+ if (result.status === 'fulfilled') {
485
+ if (result.value && addSitemapUrl(candidateSitemapUrl)) {
486
+ yield candidateSitemapUrl;
370
487
  }
371
488
  }
489
+ else {
490
+ logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
491
+ error: result.reason,
492
+ });
493
+ }
372
494
  }
373
495
  }
374
496
  };
@@ -380,12 +502,17 @@ export async function* discoverValidSitemaps(urls, options = {}) {
380
502
  }, {});
381
503
  const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
382
504
  const discoveredUrls = new Set();
383
- for await (const url of mergeAsyncIterables(...iterables)) {
384
- if (discoveredUrls.has(url)) {
385
- continue;
505
+ try {
506
+ for await (const url of mergeAsyncIterables(...iterables)) {
507
+ if (discoveredUrls.has(url)) {
508
+ continue;
509
+ }
510
+ discoveredUrls.add(url);
511
+ yield url;
386
512
  }
387
- discoveredUrls.add(url);
388
- yield url;
513
+ }
514
+ finally {
515
+ clearTimeout(timeoutHandle);
516
+ externalSignal?.removeEventListener('abort', onExternalAbort);
389
517
  }
390
518
  }
391
- //# sourceMappingURL=sitemap.js.map
@@ -19,6 +19,8 @@ export declare function emailsFromText(text: string): string[];
19
19
  /**
20
20
  * The function extracts email addresses from a list of URLs.
21
21
  * Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
22
+ * A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
23
+ * such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
22
24
  * Note that the function preserves the order of emails and keep duplicates.
23
25
  * @param urls Array of URLs.
24
26
  * @return Array of emails addresses found.
@@ -225,7 +227,7 @@ export declare const TWITTER_REGEX: RegExp;
225
227
  * ```
226
228
  * import { social } from 'crawlee';
227
229
  *
228
- * const matches = text.match(social.TWITTER_REGEX_STRING);
230
+ * const matches = text.match(social.TWITTER_REGEX_GLOBAL);
229
231
  * if (matches) console.log(`${matches.length} Twitter profiles found!`);
230
232
  * ```
231
233
  */
@@ -488,5 +490,4 @@ export declare const DISCORD_REGEX_GLOBAL: RegExp;
488
490
  * so that the caller doesn't need to parse the HTML document again, if needed.
489
491
  * @return An object with the social handles.
490
492
  */
491
- export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles;
492
- //# sourceMappingURL=social.d.ts.map
493
+ export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;