@crawlee/utils 4.0.0-rc.0 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  import type { BaseHttpClient } from '@crawlee/http-client';
2
- import type { CrawleeLogger } from '@crawlee/types';
2
+ import type { CrawleeLogger, ISession } from '@crawlee/types';
3
3
  import { Sitemap } from './sitemap.js';
4
4
  import { type EnqueueStrategy } from './url.js';
5
5
  export interface RobotsTxtFileSitemapsOptions {
@@ -38,12 +38,14 @@ export declare class RobotsTxtFile {
38
38
  * @param [options.signal] an AbortSignal to cancel the request
39
39
  * @param [options.timeoutMillis] timeout in milliseconds for the request
40
40
  * @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
41
+ * @param [options.session] a session whose fingerprint and cookies the request uses, like a page request would
41
42
  */
42
43
  static find(url: string, options?: {
43
44
  signal?: AbortSignal;
44
45
  timeoutMillis?: number;
45
46
  proxyUrl?: string;
46
47
  httpClient?: BaseHttpClient;
48
+ session?: ISession;
47
49
  logger?: CrawleeLogger;
48
50
  }): Promise<RobotsTxtFile>;
49
51
  /**
@@ -53,7 +55,6 @@ export declare class RobotsTxtFile {
53
55
  * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
54
56
  */
55
57
  static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
56
- private static load;
57
58
  /**
58
59
  * Get crawl delay for a given user agent.
59
60
  * @param [userAgent] relevant user agent, default to `*`
@@ -38,12 +38,13 @@ export class RobotsTxtFile {
38
38
  * @param [options.signal] an AbortSignal to cancel the request
39
39
  * @param [options.timeoutMillis] timeout in milliseconds for the request
40
40
  * @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
41
+ * @param [options.session] a session whose fingerprint and cookies the request uses, like a page request would
41
42
  */
42
43
  static async find(url, options) {
43
44
  const robotsTxtFileUrl = new URL(url);
44
45
  robotsTxtFileUrl.pathname = '/robots.txt';
45
46
  robotsTxtFileUrl.search = '';
46
- return RobotsTxtFile.load(robotsTxtFileUrl.toString(), options);
47
+ return RobotsTxtFile.#load(robotsTxtFileUrl.toString(), options);
47
48
  }
48
49
  /**
49
50
  * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
@@ -55,20 +56,20 @@ export class RobotsTxtFile {
55
56
  // @ts-ignore
56
57
  return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
57
58
  }
58
- static async load(url, options) {
59
- const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
59
+ static async #load(url, options) {
60
+ const { proxyUrl, logger, session, httpClient = new FetchHttpClient() } = options || {};
60
61
  const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
62
+ session,
61
63
  proxyUrl,
62
64
  timeoutMillis: options?.timeoutMillis,
63
65
  signal: options?.signal,
64
66
  });
65
- if (response.status < 200 || response.status >= 300) {
66
- throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
67
- }
68
- if (response.status === 404) {
67
+ // RFC 9309: an unavailable (4xx) robots.txt allows everything, an unreachable (5xx) one disallows everything.
68
+ if (response.status >= 400 && response.status < 600) {
69
+ const allowed = response.status < 500;
69
70
  return new RobotsTxtFile(url, {
70
71
  isAllowed() {
71
- return true;
72
+ return allowed;
72
73
  },
73
74
  getSitemaps() {
74
75
  return [];
@@ -78,6 +79,9 @@ export class RobotsTxtFile {
78
79
  },
79
80
  }, proxyUrl, logger);
80
81
  }
82
+ if (response.status < 200 || response.status >= 300) {
83
+ throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
84
+ }
81
85
  // @ts-ignore
82
86
  return new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl, logger);
83
87
  }
@@ -30,7 +30,7 @@ export declare function objectWithKeys(keys: string[], message?: string): z.ZodT
30
30
  * Accepts only instances of {@link BaseHttpClient} (all Crawlee HTTP clients extend it).
31
31
  * @internal
32
32
  */
33
- export declare const httpClient: z.ZodCustom<BaseHttpClient, BaseHttpClient>;
33
+ export declare const httpClient: z.ZodInstanceOf<BaseHttpClient>;
34
34
  /**
35
35
  * Accepts any object implementing the CrawleeLogger interface.
36
36
  * @internal
@@ -47,7 +47,7 @@ export declare const typedArray: z.ZodCustom<NodeJS.TypedArray<ArrayBufferLike>,
47
47
  */
48
48
  export declare const plainObject: z.ZodCustom<Record<string, unknown>, Record<string, unknown>>;
49
49
  /**
50
- * Shape of a request stored in a request queue.
50
+ * Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
51
51
  * @internal
52
52
  */
53
53
  export declare const storageRequest: z.ZodObject<{
@@ -54,11 +54,7 @@ export const typedArray = z.custom((value) => ArrayBuffer.isView(value) && !(val
54
54
  * @internal
55
55
  */
56
56
  export const plainObject = z.custom((value) => typeof value === 'object' && value !== null && !Array.isArray(value), { message: 'Invalid input: expected an object' });
57
- /**
58
- * Shape of a request stored in a request queue.
59
- * @internal
60
- */
61
- export const storageRequest = z.looseObject({
57
+ const storageRequestShape = z.looseObject({
62
58
  id: z.string(),
63
59
  url: z.url({ protocol: /^https?$/ }),
64
60
  uniqueKey: z.string(),
@@ -66,11 +62,16 @@ export const storageRequest = z.looseObject({
66
62
  retryCount: z.number().int().optional(),
67
63
  handledAt: z.union([z.string(), z.date()]).optional(),
68
64
  });
65
+ /**
66
+ * Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
67
+ * @internal
68
+ */
69
+ export const storageRequest = z.compile(storageRequestShape);
69
70
  /**
70
71
  * {@link storageRequest} before an id is assigned.
71
72
  * @internal
72
73
  */
73
- export const storageRequestWithoutId = storageRequest.omit({ id: true });
74
+ export const storageRequestWithoutId = z.compile(storageRequestShape.omit({ id: true }));
74
75
  /**
75
76
  * `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —
76
77
  * instead of zod's bare `expected array`. Element failures keep zod's per-index messages, and `elements`
@@ -86,7 +87,7 @@ export function arrayOf(item, elements) {
86
87
  * Batch of {@link storageRequestWithoutId}.
87
88
  * @internal
88
89
  */
89
- export const storageRequestBatch = arrayOf(storageRequestWithoutId, 'requests');
90
+ export const storageRequestBatch = z.compile(arrayOf(storageRequestWithoutId, 'requests'));
90
91
  /**
91
92
  * Options of request queue add/update operations.
92
93
  * @internal
@@ -82,6 +82,7 @@ export declare function parseSitemap<T extends ParseSitemapOptions>(initialSourc
82
82
  * ```
83
83
  */
84
84
  export declare class Sitemap {
85
+ #private;
85
86
  readonly urls: string[];
86
87
  constructor(urls: string[]);
87
88
  /**
@@ -103,7 +104,6 @@ export declare class Sitemap {
103
104
  * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
104
105
  */
105
106
  static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
106
- private static parse;
107
107
  }
108
108
  /**
109
109
  * Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
@@ -3,7 +3,7 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
3
3
  import { StringDecoder } from 'node:string_decoder';
4
4
  import { createGunzip } from 'node:zlib';
5
5
  import { FetchHttpClient } from '@crawlee/http-client';
6
- import MIMEType from 'whatwg-mimetype';
6
+ import { MIMEType } from 'whatwg-mimetype';
7
7
  import { mergeAsyncIterables } from './iterables.js';
8
8
  import { RobotsTxtFile } from './robots.js';
9
9
  import { filterUrl } from './url.js';
@@ -14,33 +14,23 @@ class SitemapTxtParser extends Transform {
14
14
  super({
15
15
  readableObjectMode: true,
16
16
  transform: (chunk, _encoding, callback) => {
17
- this.processBuffer(this.#decoder.write(chunk), false);
17
+ this.#processBuffer(this.#decoder.write(chunk), false);
18
18
  callback();
19
19
  },
20
20
  flush: (callback) => {
21
- this.processBuffer(this.#decoder.end(), true);
21
+ this.#processBuffer(this.#decoder.end(), true);
22
22
  callback();
23
23
  },
24
24
  });
25
25
  }
26
- processBuffer(input, finalize) {
27
- this.#buffer += input;
28
- if (finalize || this.#buffer.includes('\n')) {
29
- const parts = this.#buffer
30
- .split('\n')
31
- .map((part) => part.trim())
32
- .filter((part) => part.length > 0);
33
- if (finalize) {
34
- for (const url of parts) {
35
- this.push({ type: 'url', loc: url });
36
- }
37
- this.#buffer = '';
38
- }
39
- else if (parts.length > 0) {
40
- for (const url of parts.slice(0, -1)) {
41
- this.push({ type: 'url', loc: url });
42
- }
43
- this.#buffer = parts.at(-1);
26
+ #processBuffer(input, finalize) {
27
+ const lines = (this.#buffer + input).split('\n');
28
+ // The text after the last line break may be a partial line, so it waits for the next chunk.
29
+ this.#buffer = finalize ? '' : lines.pop();
30
+ for (const line of lines) {
31
+ const url = line.trim();
32
+ if (url.length > 0) {
33
+ this.push({ type: 'url', loc: url });
44
34
  }
45
35
  }
46
36
  }
@@ -52,7 +42,7 @@ class SitemapXmlParser extends Transform {
52
42
  #currentTag = undefined;
53
43
  #url = {};
54
44
  static async create() {
55
- const { SAXParser } = await import('sax');
45
+ const { SAXParser } = (await import('sax')).default;
56
46
  return new SitemapXmlParser(new SAXParser(true));
57
47
  }
58
48
  constructor(parser) {
@@ -72,13 +62,13 @@ class SitemapXmlParser extends Transform {
72
62
  },
73
63
  });
74
64
  this.#parser = parser;
75
- this.#parser.onopentag = this.onOpenTag.bind(this);
76
- this.#parser.onclosetag = this.onCloseTag.bind(this);
77
- this.#parser.ontext = this.onText.bind(this);
78
- this.#parser.oncdata = this.onText.bind(this);
65
+ this.#parser.onopentag = this.#onOpenTag.bind(this);
66
+ this.#parser.onclosetag = this.#onCloseTag.bind(this);
67
+ this.#parser.ontext = this.#onText.bind(this);
68
+ this.#parser.oncdata = this.#onText.bind(this);
79
69
  this.#parser.onerror = this.destroy.bind(this);
80
70
  }
81
- onOpenTag(node) {
71
+ #onOpenTag(node) {
82
72
  if (this.#rootTagName !== undefined) {
83
73
  if (node.name === 'loc' ||
84
74
  node.name === 'lastmod' ||
@@ -94,7 +84,7 @@ class SitemapXmlParser extends Transform {
94
84
  this.#rootTagName = 'sitemapindex';
95
85
  }
96
86
  }
97
- onCloseTag(name) {
87
+ #onCloseTag(name) {
98
88
  if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
99
89
  this.#currentTag = undefined;
100
90
  }
@@ -105,17 +95,21 @@ class SitemapXmlParser extends Transform {
105
95
  this.#url = {};
106
96
  }
107
97
  }
108
- onText(text) {
98
+ #onText(text) {
99
+ text = text.trim();
100
+ // Whitespace around a CDATA section arrives as separate text events.
101
+ if (text === '') {
102
+ return;
103
+ }
109
104
  if (this.#currentTag === 'loc') {
110
105
  if (this.#rootTagName === 'sitemapindex') {
111
- this.push({ type: 'sitemapUrl', url: text.trim() });
106
+ this.push({ type: 'sitemapUrl', url: text });
112
107
  }
113
108
  if (this.#rootTagName === 'urlset') {
114
109
  this.#url ??= {};
115
- this.#url.loc = text.trim();
110
+ this.#url.loc = text;
116
111
  }
117
112
  }
118
- text = text.trim();
119
113
  if (this.#currentTag === 'lastmod') {
120
114
  const lastmod = new Date(text);
121
115
  if (!Number.isNaN(lastmod.getTime())) {
@@ -150,6 +144,10 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
150
144
  if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
151
145
  return new SitemapTxtParser();
152
146
  }
147
+ // A name like `sitemap.gz` says nothing about what was compressed. Sitemaps are XML unless named otherwise.
148
+ if (mimeType?.essence === 'application/gzip') {
149
+ return SitemapXmlParser.create();
150
+ }
153
151
  throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);
154
152
  };
155
153
  while (sources.length > 0) {
@@ -241,44 +239,66 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
241
239
  // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
242
240
  // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
243
241
  let droppedUrlEntries = 0;
244
- for await (const item of items) {
245
- if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
246
- if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
247
- logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
248
- continue;
242
+ const iterator = items[Symbol.asyncIterator]();
243
+ try {
244
+ while (true) {
245
+ let next;
246
+ try {
247
+ next = await iterator.next();
249
248
  }
250
- // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
251
- // parent. Raw string sources have no parent URL, so the check is skipped.
252
- if (source.type === 'url') {
253
- const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
254
- if (!allowed) {
255
- logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
256
- continue;
249
+ catch (e) {
250
+ // Parsing failed: give up on this sitemap, keep the others.
251
+ // Raw sources already logged the error in their pipeline callback.
252
+ if (source.type !== 'raw') {
253
+ logger?.warning(`Malformed sitemap content: ${source.url} (${e})`);
257
254
  }
255
+ break;
258
256
  }
259
- sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
260
- if (emitNestedSitemaps) {
261
- yield { loc: item.url, originSitemapUrl: null };
262
- }
263
- }
264
- if (item.type === 'url') {
265
- // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
266
- if (source.type === 'url') {
267
- const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
268
- if (!allowed) {
269
- droppedUrlEntries++;
270
- logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
257
+ if (next.done)
258
+ break;
259
+ const item = next.value;
260
+ if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
261
+ if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
262
+ logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
271
263
  continue;
272
264
  }
265
+ // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
266
+ // parent. Raw string sources have no parent URL, so the check is skipped.
267
+ if (source.type === 'url') {
268
+ const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
269
+ if (!allowed) {
270
+ logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
271
+ continue;
272
+ }
273
+ }
274
+ sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
275
+ if (emitNestedSitemaps) {
276
+ yield { loc: item.url, originSitemapUrl: null };
277
+ }
278
+ }
279
+ if (item.type === 'url') {
280
+ // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
281
+ if (source.type === 'url') {
282
+ const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
283
+ if (!allowed) {
284
+ droppedUrlEntries++;
285
+ logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
286
+ continue;
287
+ }
288
+ }
289
+ yield {
290
+ ...item,
291
+ originSitemapUrl: source.type === 'url'
292
+ ? source.url
293
+ : `raw://${createHash('sha256').update(source.content).digest('base64')}`,
294
+ };
273
295
  }
274
- yield {
275
- ...item,
276
- originSitemapUrl: source.type === 'url'
277
- ? source.url
278
- : `raw://${createHash('sha256').update(source.content).digest('base64')}`,
279
- };
280
296
  }
281
297
  }
298
+ finally {
299
+ // Mirrors `for await` cleanup: destroys the stream when we stop early or the consumer bails out.
300
+ await iterator.return?.();
301
+ }
282
302
  if (droppedUrlEntries > 0 && source.type === 'url') {
283
303
  logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
284
304
  }
@@ -323,7 +343,7 @@ export class Sitemap {
323
343
  * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
324
344
  */
325
345
  static async load(urls, proxyUrl, parseSitemapOptions) {
326
- return await this.parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
346
+ return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
327
347
  }
328
348
  /**
329
349
  * Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
@@ -331,9 +351,9 @@ export class Sitemap {
331
351
  * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
332
352
  */
333
353
  static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
334
- return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
354
+ return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
335
355
  }
336
- static async parse(sources, proxyUrl, parseSitemapOptions) {
356
+ static async #parse(sources, proxyUrl, parseSitemapOptions) {
337
357
  const urls = [];
338
358
  try {
339
359
  for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
@@ -423,6 +443,11 @@ export async function* discoverValidSitemaps(urls, options = {}) {
423
443
  }
424
444
  else {
425
445
  const firstUrl = new URL(domainUrls[0]);
446
+ // The guessed candidates are well-known paths on the same origin, so the query string
447
+ // and fragment of the input URL are not part of them - carrying them over probes (and
448
+ // yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
449
+ firstUrl.search = '';
450
+ firstUrl.hash = '';
426
451
  const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
427
452
  const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
428
453
  firstUrl.pathname = pathname;
@@ -19,6 +19,8 @@ export declare function emailsFromText(text: string): string[];
19
19
  /**
20
20
  * The function extracts email addresses from a list of URLs.
21
21
  * Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
22
+ * A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
23
+ * such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
22
24
  * Note that the function preserves the order of emails and keep duplicates.
23
25
  * @param urls Array of URLs.
24
26
  * @return Array of emails addresses found.
@@ -225,7 +227,7 @@ export declare const TWITTER_REGEX: RegExp;
225
227
  * ```
226
228
  * import { social } from 'crawlee';
227
229
  *
228
- * const matches = text.match(social.TWITTER_REGEX_STRING);
230
+ * const matches = text.match(social.TWITTER_REGEX_GLOBAL);
229
231
  * if (matches) console.log(`${matches.length} Twitter profiles found!`);
230
232
  * ```
231
233
  */
@@ -30,6 +30,8 @@ export function emailsFromText(text) {
30
30
  /**
31
31
  * The function extracts email addresses from a list of URLs.
32
32
  * Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
33
+ * A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
34
+ * such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
33
35
  * Note that the function preserves the order of emails and keep duplicates.
34
36
  * @param urls Array of URLs.
35
37
  * @return Array of emails addresses found.
@@ -44,9 +46,20 @@ export function emailsFromUrls(urls) {
44
46
  continue;
45
47
  if (!EMAIL_URL_PREFIX_REGEX.test(url))
46
48
  continue;
47
- const email = url.replace(EMAIL_URL_PREFIX_REGEX, '').trim();
48
- if (EMAIL_REGEX.test(email))
49
- emails.push(email);
49
+ // Characters like `?` and `,` inside an address must be percent-encoded, so it's safe
50
+ // to split on them before decoding.
51
+ const [addresses] = url.replace(EMAIL_URL_PREFIX_REGEX, '').split('?');
52
+ for (const address of addresses.split(',')) {
53
+ let email = address.trim();
54
+ try {
55
+ email = decodeURIComponent(email).trim();
56
+ }
57
+ catch {
58
+ // Keep the raw value if it isn't valid percent-encoding
59
+ }
60
+ if (EMAIL_REGEX.test(email))
61
+ emails.push(email);
62
+ }
50
63
  }
51
64
  return emails;
52
65
  }
@@ -138,8 +151,18 @@ export function phonesFromUrls(urls) {
138
151
  continue;
139
152
  if (!PHONE_URL_PREFIX_REGEX.test(url))
140
153
  continue;
141
- const phone = url.replace(PHONE_URL_PREFIX_REGEX, '').trim();
142
- if (PHONE_REGEX.test(phone))
154
+ // Per RFC 3966, parameters such as `;ext=89` can follow the number, and `-`, `.`, `(` and `)`
155
+ // are only visual separators. They are checked without separators too, so a one-digit country
156
+ // code like in `tel:+1-555-123-4567` is accepted.
157
+ const [number] = url.replace(PHONE_URL_PREFIX_REGEX, '').split(';');
158
+ let phone = number.trim();
159
+ try {
160
+ phone = decodeURIComponent(phone).trim();
161
+ }
162
+ catch {
163
+ // Keep the raw value if it isn't valid percent-encoding
164
+ }
165
+ if (PHONE_REGEX.test(phone) || PHONE_REGEX.test(phone.replace(/[-.() ]/g, '')))
143
166
  phones.push(phone);
144
167
  }
145
168
  return phones;
@@ -326,7 +349,7 @@ export const TWITTER_REGEX = new RegExp(`^${TWITTER_REGEX_STRING}$`, 'i');
326
349
  * ```
327
350
  * import { social } from 'crawlee';
328
351
  *
329
- * const matches = text.match(social.TWITTER_REGEX_STRING);
352
+ * const matches = text.match(social.TWITTER_REGEX_GLOBAL);
330
353
  * if (matches) console.log(`${matches.length} Twitter profiles found!`);
331
354
  * ```
332
355
  */
@@ -590,7 +613,7 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
590
613
  * @return An object with the social handles.
591
614
  */
592
615
  export async function parseHandlesFromHtml(html, data = null) {
593
- const cheerio = await import('cheerio');
616
+ const cheerio = await import('cheerio/slim');
594
617
  const result = {
595
618
  emails: [],
596
619
  phones: [],
@@ -54,10 +54,8 @@ export declare enum EnqueueStrategy {
54
54
  export declare const UNSUPPORTED_SCHEME_MESSAGE = "unsupported URL scheme (only http and https are allowed)";
55
55
  /**
56
56
  * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
57
- * considered here (use {@link filterUrl} for the combined scheme + strategy check).
58
- *
59
- * The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
60
- * (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
57
+ * considered here (use {@link filterUrl} for the combined scheme + strategy check). This is the single
58
+ * definition of the strategies, shared by `enqueueLinks`, sitemaps and robots.txt.
61
59
  */
62
60
  export declare function matchesEnqueueStrategy(strategy: EnqueueStrategy | `${EnqueueStrategy}`, target: URL, origin: URL): boolean;
63
61
  /**
package/internals/url.js CHANGED
@@ -71,10 +71,8 @@ function normalizeHostname(hostname) {
71
71
  }
72
72
  /**
73
73
  * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
74
- * considered here (use {@link filterUrl} for the combined scheme + strategy check).
75
- *
76
- * The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
77
- * (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
74
+ * considered here (use {@link filterUrl} for the combined scheme + strategy check). This is the single
75
+ * definition of the strategies, shared by `enqueueLinks`, sitemaps and robots.txt.
78
76
  */
79
77
  export function matchesEnqueueStrategy(strategy, target, origin) {
80
78
  switch (strategy) {
@@ -87,8 +85,8 @@ export function matchesEnqueueStrategy(strategy, target, origin) {
87
85
  if (originDomain) {
88
86
  return originDomain === getDomain(target.hostname, { mixedInputs: false });
89
87
  }
90
- // No registrable domain (e.g. an IP address), fall back to comparing origins.
91
- return target.origin === origin.origin;
88
+ // No registrable domain (e.g. an IP address or `localhost`), fall back to the same hostname.
89
+ return matchesEnqueueStrategy('same-hostname', target, origin);
92
90
  }
93
91
  case 'same-origin':
94
92
  // Compare scheme/host/port directly so a trailing-dot host is normalized.
@@ -1,25 +1,9 @@
1
- import type { z } from 'zod';
1
+ import type { Dictionary } from '@crawlee/types';
2
+ export { ArgumentValidationError, parseArgument } from '@apify/validations';
2
3
  /**
3
- * Thrown when an argument fails schema validation.
4
- *
5
- * Its `message` is a human-readable sentence naming the offending field and the
6
- * value it received (rather than a raw JSON dump). The structured
7
- * {@link https://zod.dev | zod} issues are available on `issues`, and the
8
- * original `ZodError` on `cause`, for programmatic inspection.
9
- */
10
- export declare class ArgumentValidationError extends Error {
11
- /** Structured issues from the underlying schema check. */
12
- readonly issues: z.ZodError['issues'];
13
- /** The raw zod error that triggered this. */
14
- readonly cause: z.ZodError;
15
- constructor(error: z.ZodError, value: unknown, label?: string);
16
- }
17
- /**
18
- * Parses `value` with `schema`, returning the typed result (with schema defaults applied).
19
- * Throws {@link ArgumentValidationError} on failure.
20
- *
21
- * The optional `label` names the interface being validated and is appended to every error line
22
- * (e.g. ``… at `maxRequestRetries` in `BasicCrawlerOptions` ``).
4
+ * Rejects options that exist only to configure the browser pool the crawler would have built for itself.
5
+ * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
6
+ * into a second, half-working way of configuring the same pool.
23
7
  * @internal
24
8
  */
25
- export declare function parseArgument<TValue, TSchema extends z.ZodType>(value: TValue, schema: TSchema, label?: string): TValue & z.output<TSchema>;
9
+ export declare function assertBrowserPoolNotConfigured(crawlerName: string, ignoredOptions: Dictionary): void;