@crawlee/utils 4.0.0-rc.1 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.d.ts CHANGED
@@ -5,7 +5,7 @@ export type { DownloadListOfUrlsOptions, ExtractUrlsOptions } from './internals/
5
5
  export { sleep } from './internals/general.js';
6
6
  export * as social from './internals/social.js';
7
7
  export * from './internals/extract-microdata.js';
8
- export * from './internals/open_graph_parser.js';
8
+ export * from './internals/open-graph-parser.js';
9
9
  export * from './internals/robots.js';
10
10
  export { discoverValidSitemaps, Sitemap } from './internals/sitemap.js';
11
11
  export type { ParseSitemapOptions } from './internals/sitemap.js';
package/index.js CHANGED
@@ -4,7 +4,7 @@ export { EnqueueStrategy } from './internals/url.js';
4
4
  export { sleep } from './internals/general.js';
5
5
  export * as social from './internals/social.js';
6
6
  export * from './internals/extract-microdata.js';
7
- export * from './internals/open_graph_parser.js';
7
+ export * from './internals/open-graph-parser.js';
8
8
  export * from './internals/robots.js';
9
9
  export { discoverValidSitemaps, Sitemap } from './internals/sitemap.js';
10
10
  export { ArgumentValidationError } from './internals/validation.js';
package/internal.d.ts CHANGED
@@ -3,7 +3,9 @@ export type { CheerioAPI, Cheerio, Element } from './internals/cheerio.js';
3
3
  export { extractUrlsFromCheerio } from './internals/cheerio.js';
4
4
  export { tryAbsoluteURL } from './internals/extract-urls.js';
5
5
  export { expandShadowRoots, URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
6
+ export { processHttpRequestOptions } from './internals/http-request.js';
6
7
  export * from './internals/iterables.js';
8
+ export { MutationJournal } from './internals/mutation-journal.js';
7
9
  export * from './internals/url.js';
8
10
  export * from './internals/validation.js';
9
11
  export * as schemas from './internals/schemas.js';
package/internal.js CHANGED
@@ -2,7 +2,9 @@ export * from './internals/blocked.js';
2
2
  export { extractUrlsFromCheerio } from './internals/cheerio.js';
3
3
  export { tryAbsoluteURL } from './internals/extract-urls.js';
4
4
  export { expandShadowRoots, URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
5
+ export { processHttpRequestOptions } from './internals/http-request.js';
5
6
  export * from './internals/iterables.js';
7
+ export { MutationJournal } from './internals/mutation-journal.js';
6
8
  export * from './internals/url.js';
7
9
  export * from './internals/validation.js';
8
10
  export * as schemas from './internals/schemas.js';
@@ -1,4 +1,11 @@
1
1
  export declare const CLOUDFLARE_RETRY_CSS_SELECTORS: string[];
2
+ /**
3
+ * The AWS WAF interstitials: the JS challenge, served with a `202` status, and the CAPTCHA it escalates to, served with a `405`.
4
+ * Neither status is a blocked one. Matching the container as a direct child of `<body>` keeps regular pages
5
+ * that load the AWS WAF integration scripts, or embed the CAPTCHA into their own markup, from matching.
6
+ * The `:has()` argument must not start with `>`, Playwright's selector engine never matches it on the root element.
7
+ */
8
+ export declare const AWS_WAF_RETRY_CSS_SELECTORS: string[];
2
9
  /**
3
10
  * CSS selectors for elements that should trigger a retry, as the crawler is likely getting blocked.
4
11
  */
@@ -1,9 +1,20 @@
1
1
  export const CLOUDFLARE_RETRY_CSS_SELECTORS = ['#turnstile-wrapper iframe[src^="https://challenges.cloudflare.com"]'];
2
+ /**
3
+ * The AWS WAF interstitials: the JS challenge, served with a `202` status, and the CAPTCHA it escalates to, served with a `405`.
4
+ * Neither status is a blocked one. Matching the container as a direct child of `<body>` keeps regular pages
5
+ * that load the AWS WAF integration scripts, or embed the CAPTCHA into their own markup, from matching.
6
+ * The `:has()` argument must not start with `>`, Playwright's selector engine never matches it on the root element.
7
+ */
8
+ export const AWS_WAF_RETRY_CSS_SELECTORS = [
9
+ 'html:has(head > script[src*=".awswaf.com/"][src$="/challenge.js"]) > body > #challenge-container',
10
+ 'html:has(head > script[src*=".awswaf.com/"][src$="/captcha.js"]) > body > #captcha-container',
11
+ ];
2
12
  /**
3
13
  * CSS selectors for elements that should trigger a retry, as the crawler is likely getting blocked.
4
14
  */
5
15
  export const RETRY_CSS_SELECTORS = [
6
16
  ...CLOUDFLARE_RETRY_CSS_SELECTORS,
17
+ ...AWS_WAF_RETRY_CSS_SELECTORS,
7
18
  'div#infoDiv0 a[href*="//www.google.com/policies/terms/"]',
8
19
  'iframe[src*="_Incapsula_Resource"]',
9
20
  ];
@@ -1,7 +1,7 @@
1
1
  import { tryAbsoluteURL } from './extract-urls.js';
2
2
  // NOTE: We are skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.
3
3
  const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;
4
- const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|div|fieldset|form|table|tr|select|option)$/i;
4
+ const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|dt|dd|div|fieldset|legend|form|table|caption|tr|select|option|hr|article|aside|section|nav|header|footer|main|hgroup|figure|figcaption|details|summary|dialog|menu)$/i;
5
5
  /**
6
6
  * The function converts a HTML document to a plain text.
7
7
  *
@@ -34,50 +34,59 @@ export async function htmlToText(htmlOrCheerioElement) {
34
34
  if (!htmlOrCheerioElement)
35
35
  return '';
36
36
  const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : load(htmlOrCheerioElement);
37
- let text = '';
38
- const process = (elems) => {
37
+ // Never read the accumulated text mid-loop: each read flattens V8's cons-string, making the traversal quadratic
38
+ const parts = [];
39
+ let last = '';
40
+ const push = (part) => {
41
+ if (!part)
42
+ return;
43
+ parts.push(part);
44
+ last = part[part.length - 1];
45
+ };
46
+ const process = (elems, inPre = false) => {
39
47
  const len = elems ? elems.length : 0;
40
48
  for (let i = 0; i < len; i++) {
41
49
  const elem = elems[i];
42
50
  if (elem.type === 'text') {
43
- // Compress spaces, unless we're inside <pre> element
51
+ // Compress spaces, unless we're inside <pre> element (including nested elements like <pre><code>)
44
52
  let compr;
45
- if (elem.parent?.tagName === 'pre') {
53
+ if (inPre) {
46
54
  // Strip single newline right after `<pre>` that htmlparser2 keeps
47
- compr = elem.parent.children?.[0] === elem ? elem.data.replace(/^\r?\n/, '') : elem.data;
55
+ const isFirstInPre = elem.parent?.tagName === 'pre' && elem.parent.children?.[0] === elem;
56
+ compr = isFirstInPre ? elem.data.replace(/^\r?\n/, '') : elem.data;
48
57
  }
49
58
  else
50
59
  compr = elem.data.replace(/\s+/g, ' ');
51
60
  // If text is empty or ends with a whitespace, don't add the leading whitespace
52
- if (compr.startsWith(' ') && /(^|\s)$/.test(text))
61
+ if (!inPre && compr.startsWith(' ') && (last === '' || /\s/.test(last)))
53
62
  compr = compr.substring(1);
54
- text += compr;
63
+ push(compr);
55
64
  }
56
65
  else if (elem.type === 'comment' || SKIP_TAGS_REGEX.test(elem.tagName)) {
57
66
  // Skip comments and special elements
58
67
  }
59
68
  else if (elem.tagName === 'br') {
60
- text += '\n';
69
+ push('\n');
61
70
  }
62
- else if (elem.tagName === 'td') {
63
- process(elem.children);
64
- text += '\t';
71
+ else if (elem.tagName === 'td' || elem.tagName === 'th') {
72
+ process(elem.children, inPre);
73
+ push('\t');
65
74
  }
66
75
  else {
67
76
  // Block elements must be surrounded by newlines (unless beginning of text)
68
77
  const isBlockTag = BLOCK_TAGS_REGEX.test(elem.tagName);
69
- if (isBlockTag && !/(^|\n)$/.test(text))
70
- text += '\n';
71
- process(elem.children);
72
- if (isBlockTag && !text.endsWith('\n'))
73
- text += '\n';
78
+ if (isBlockTag && last !== '' && last !== '\n')
79
+ push('\n');
80
+ process(elem.children, inPre || elem.tagName === 'pre');
81
+ if (isBlockTag && last !== '\n')
82
+ push('\n');
74
83
  }
75
84
  }
76
85
  };
77
86
  // If HTML document has body, only convert that, otherwise convert the entire HTML
78
87
  const $body = $('body');
79
88
  process($body.length > 0 ? $body : $.root());
80
- return text.trim();
89
+ return parts.join('').trim();
81
90
  }
82
91
  /**
83
92
  * Extracts URLs from a given Cheerio object.
@@ -31,26 +31,30 @@ function parseItem(context, element, ancestors) {
31
31
  }
32
32
  function collectPropertyElements(context, scope) {
33
33
  const elements = [];
34
- collectFromNodes(scope.children, elements);
34
+ // An `itemref` can point at an element already reached from the item's own subtree, or at one that
35
+ // contains another referenced element. The spec visits every element once, so each property counts once.
36
+ const visited = new Set();
37
+ collectFromNodes(scope.children, elements, visited);
35
38
  for (const id of uniqueTokens(scope.attribs.itemref)) {
36
39
  const referenced = (context.idIndex ??= indexIds(context.$)).get(id);
37
40
  if (referenced) {
38
- collectFromNodes([referenced], elements);
41
+ collectFromNodes([referenced], elements, visited);
39
42
  }
40
43
  }
41
44
  return elements;
42
45
  }
43
- function collectFromNodes(nodes, elements) {
46
+ function collectFromNodes(nodes, elements, visited) {
44
47
  for (const node of nodes) {
45
- if (!isTag(node)) {
48
+ if (!isTag(node) || visited.has(node)) {
46
49
  continue;
47
50
  }
51
+ visited.add(node);
48
52
  if ('itemprop' in node.attribs) {
49
53
  elements.push(node);
50
54
  }
51
55
  // A nested item owns everything below it, so its subtree is not part of the enclosing item.
52
56
  if (!('itemscope' in node.attribs)) {
53
- collectFromNodes(node.children, elements);
57
+ collectFromNodes(node.children, elements, visited);
54
58
  }
55
59
  }
56
60
  }
@@ -20,13 +20,7 @@ const extractUrlsOptionsSchema = z.strictObject({
20
20
  */
21
21
  export async function downloadListOfUrls(options) {
22
22
  const { url, encoding, urlRegExp, proxyUrl, httpClient } = parseArgument(options, downloadListOfUrlsOptionsSchema);
23
- // Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one.
24
- const match = /^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/.exec(url);
25
- let fixedUrl = url;
26
- if (match) {
27
- fixedUrl = `${match[1]}/gviz/tq?tqx=out:csv`;
28
- }
29
- const response = await httpClient.sendRequest(new Request(fixedUrl, { method: 'GET' }), {
23
+ const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
30
24
  proxyUrl,
31
25
  });
32
26
  const string = new TextDecoder(encoding).decode(new Uint8Array(await response.arrayBuffer()));
@@ -0,0 +1,5 @@
1
+ import type { HttpRequest, HttpRequestOptions } from '@crawlee/types';
2
+ /**
3
+ * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
4
+ */
5
+ export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
@@ -0,0 +1,36 @@
1
+ import { Readable } from 'node:stream';
2
+ import { applySearchParams } from './url.js';
3
+ /**
4
+ * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
5
+ */
6
+ export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
7
+ const url = new URL(request.url);
8
+ const headers = new Headers(request.headers);
9
+ applySearchParams(url, searchParams);
10
+ if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
11
+ throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
12
+ }
13
+ const body = (() => {
14
+ if (form !== undefined) {
15
+ return Readable.from(new URLSearchParams(form).toString());
16
+ }
17
+ if (json !== undefined) {
18
+ return Readable.from(JSON.stringify(json));
19
+ }
20
+ if (request.body !== undefined) {
21
+ return Readable.from(request.body);
22
+ }
23
+ return undefined;
24
+ })();
25
+ if (form !== undefined && !headers.has('content-type')) {
26
+ headers.set('content-type', 'application/x-www-form-urlencoded');
27
+ }
28
+ if (json !== undefined && !headers.has('content-type')) {
29
+ headers.set('content-type', 'application/json');
30
+ }
31
+ if (username !== undefined || password !== undefined) {
32
+ const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
33
+ headers.set('authorization', `Basic ${encodedAuth}`);
34
+ }
35
+ return { ...request, body, url, headers };
36
+ }
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Records property writes made through the proxies it hands out, applying them to the real objects as
3
+ * they happen, so that a losing attempt's writes can be undone afterwards - on the objects themselves
4
+ * ({@link MutationJournal.revert|`revert()`}), or on a copy of them
5
+ * ({@link MutationJournal.undoOnto|`undoOnto()`}) to reconstruct how they looked before the attempt.
6
+ *
7
+ * An undo on the live object only touches a property that still holds the value the attempt wrote - a
8
+ * value written by someone else in the meantime is left alone and reported via `onConflict`. Writes that
9
+ * bypass the proxy are invisible until they trip that guard.
10
+ */
11
+ export declare class MutationJournal {
12
+ #private;
13
+ track<T extends object>(target: T, path?: PropertyKey[]): T;
14
+ /** Stops recording. Proxies keep writing through. */
15
+ close(): void;
16
+ /** Undoes the recorded writes, newest first, and stops recording. */
17
+ revert({ onConflict }?: {
18
+ onConflict?: (key: PropertyKey) => void;
19
+ }): void;
20
+ /**
21
+ * Applies the inverse of the recorded writes onto `root`, a JSON copy of the tracked object, and returns
22
+ * it. The journal is left as it is. Restored values are JSON-cloned too, so the copy shares nothing with
23
+ * the original and non-JSON values drop out the same way on both.
24
+ *
25
+ * Entries are located by the path under which their object was *first* reached, so an object reachable
26
+ * through two paths is undone under the first one only.
27
+ */
28
+ undoOnto<T extends object>(root: T): T;
29
+ }
@@ -0,0 +1,173 @@
1
+ /**
2
+ * Only plain objects and arrays are wrapped when reached through a tracked object. Built-ins such as `Map`
3
+ * or `Set` reject a proxy receiver in their methods, and the tracked objects are JSON-shaped by contract
4
+ * anyway - such values are handed out as they are, and their mutations are not recorded.
5
+ */
6
+ const isTrackable = (value) => Array.isArray(value) ||
7
+ (typeof value === 'object' &&
8
+ value !== null &&
9
+ (Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null));
10
+ /** Every proxy any journal has handed out, keyed by the proxy. Shared, as a value read through one attempt's
11
+ * proxy may be written back through another's. */
12
+ const proxyTargets = new WeakMap();
13
+ /**
14
+ * Replaces journal proxies in `value` with the objects behind them, in place for nested ones. A value read
15
+ * through a proxy and written back (`state.x = state.x ?? {}`) would otherwise store the proxy itself, and
16
+ * every later attempt would wrap it once more.
17
+ */
18
+ function unwrap(value, seen = new Set()) {
19
+ const target = typeof value === 'object' && value !== null ? proxyTargets.get(value) : undefined;
20
+ if (target)
21
+ return target;
22
+ if (!isTrackable(value) || seen.has(value))
23
+ return value;
24
+ seen.add(value);
25
+ for (const key of Object.keys(value)) {
26
+ const inner = value[key];
27
+ const unwrapped = unwrap(inner, seen);
28
+ if (unwrapped !== inner)
29
+ value[key] = unwrapped;
30
+ }
31
+ return value;
32
+ }
33
+ /**
34
+ * Records property writes made through the proxies it hands out, applying them to the real objects as
35
+ * they happen, so that a losing attempt's writes can be undone afterwards - on the objects themselves
36
+ * ({@link MutationJournal.revert|`revert()`}), or on a copy of them
37
+ * ({@link MutationJournal.undoOnto|`undoOnto()`}) to reconstruct how they looked before the attempt.
38
+ *
39
+ * An undo on the live object only touches a property that still holds the value the attempt wrote - a
40
+ * value written by someone else in the meantime is left alone and reported via `onConflict`. Writes that
41
+ * bypass the proxy are invisible until they trip that guard.
42
+ */
43
+ export class MutationJournal {
44
+ #entries = [];
45
+ #proxies = new WeakMap();
46
+ #paths = new WeakMap();
47
+ #closed = false;
48
+ track(target, path = []) {
49
+ const cached = this.#proxies.get(target);
50
+ if (cached)
51
+ return cached;
52
+ this.#paths.set(target, path);
53
+ const entry = (key, fields) => this.#entries.push({ target, path, key, ...fields });
54
+ // Accessors run with `this` as the real object, not the proxy: class accessors may touch `#private`
55
+ // members, which throw on a proxy receiver. An accessor write is therefore journaled as its own
56
+ // entry and undone through the same setter, rather than through whatever it writes underneath.
57
+ const proxy = new Proxy(target, {
58
+ get: (object, key) => {
59
+ const value = Reflect.get(object, key);
60
+ return isTrackable(value) ? this.track(value, [...path, key]) : value;
61
+ },
62
+ set: (object, key, rawNext) => {
63
+ const next = unwrap(rawNext);
64
+ const had = Reflect.has(object, key);
65
+ const prev = Reflect.get(object, key);
66
+ this.#recordArrayTruncation(object, key, next);
67
+ if (!Reflect.set(object, key, next))
68
+ return false;
69
+ // An array's `length` is implied by its elements and never undone on its own.
70
+ if (!this.#closed && !(Array.isArray(object) && key === 'length')) {
71
+ entry(key, { had, prev, next, deleted: false });
72
+ }
73
+ return true;
74
+ },
75
+ deleteProperty: (object, key) => {
76
+ const had = Reflect.has(object, key);
77
+ const prev = Reflect.get(object, key);
78
+ if (!Reflect.deleteProperty(object, key))
79
+ return false;
80
+ if (!this.#closed && had)
81
+ entry(key, { had, prev, next: undefined, deleted: true });
82
+ return true;
83
+ },
84
+ });
85
+ this.#proxies.set(target, proxy);
86
+ proxyTargets.set(proxy, target);
87
+ return proxy;
88
+ }
89
+ /**
90
+ * Shrinking an array's `length` drops elements without a trap firing for them. They are recorded as
91
+ * deletions, from the highest index down, so the reverse replay reinserts them from the lowest up.
92
+ */
93
+ #recordArrayTruncation(object, key, next) {
94
+ if (this.#closed || !Array.isArray(object) || key !== 'length')
95
+ return;
96
+ const path = this.#paths.get(object);
97
+ for (let i = object.length - 1; i >= next; i--) {
98
+ if (i in object) {
99
+ this.#entries.push({
100
+ target: object,
101
+ path,
102
+ key: String(i),
103
+ had: true,
104
+ prev: object[i],
105
+ next: undefined,
106
+ deleted: true,
107
+ });
108
+ }
109
+ }
110
+ }
111
+ /**
112
+ * Restores one entry on `target`. Array elements are handled positionally - an added element is removed
113
+ * and a removed one reinserted, shifting the rest - so the result stays something a sequence of `push`
114
+ * and `splice` calls could have produced, even next to elements another writer added meanwhile.
115
+ */
116
+ static #undo(target, { key, had, deleted }, value) {
117
+ if (Array.isArray(target) && typeof key === 'string' && Number.isInteger(Number(key))) {
118
+ if (deleted) {
119
+ target.splice(Number(key), 0, value);
120
+ }
121
+ else if (had) {
122
+ target[Number(key)] = value;
123
+ }
124
+ else {
125
+ target.splice(Number(key), 1);
126
+ }
127
+ return;
128
+ }
129
+ if (had) {
130
+ Reflect.set(target, key, value);
131
+ }
132
+ else {
133
+ Reflect.deleteProperty(target, key);
134
+ }
135
+ }
136
+ /** Stops recording. Proxies keep writing through. */
137
+ close() {
138
+ this.#closed = true;
139
+ }
140
+ /** Undoes the recorded writes, newest first, and stops recording. */
141
+ revert({ onConflict } = {}) {
142
+ this.close();
143
+ for (let i = this.#entries.length - 1; i >= 0; i--) {
144
+ const entry = this.#entries[i];
145
+ const { target, key, next, deleted } = entry;
146
+ const untouched = deleted ? !Reflect.has(target, key) : Reflect.get(target, key) === next;
147
+ if (!untouched) {
148
+ onConflict?.(key);
149
+ continue;
150
+ }
151
+ MutationJournal.#undo(target, entry, entry.prev);
152
+ }
153
+ this.#entries.length = 0;
154
+ }
155
+ /**
156
+ * Applies the inverse of the recorded writes onto `root`, a JSON copy of the tracked object, and returns
157
+ * it. The journal is left as it is. Restored values are JSON-cloned too, so the copy shares nothing with
158
+ * the original and non-JSON values drop out the same way on both.
159
+ *
160
+ * Entries are located by the path under which their object was *first* reached, so an object reachable
161
+ * through two paths is undone under the first one only.
162
+ */
163
+ undoOnto(root) {
164
+ for (let i = this.#entries.length - 1; i >= 0; i--) {
165
+ const entry = this.#entries[i];
166
+ const target = entry.path.reduce((object, step) => object?.[step], root);
167
+ if (typeof target !== 'object' || target === null)
168
+ continue;
169
+ MutationJournal.#undo(target, entry, isTrackable(entry.prev) ? JSON.parse(JSON.stringify(entry.prev)) : entry.prev);
170
+ }
171
+ return root;
172
+ }
173
+ }
@@ -1,5 +1,5 @@
1
1
  import type { BaseHttpClient } from '@crawlee/http-client';
2
- import type { CrawleeLogger } from '@crawlee/types';
2
+ import type { CrawleeLogger, ISession } from '@crawlee/types';
3
3
  import { Sitemap } from './sitemap.js';
4
4
  import { type EnqueueStrategy } from './url.js';
5
5
  export interface RobotsTxtFileSitemapsOptions {
@@ -38,12 +38,14 @@ export declare class RobotsTxtFile {
38
38
  * @param [options.signal] an AbortSignal to cancel the request
39
39
  * @param [options.timeoutMillis] timeout in milliseconds for the request
40
40
  * @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
41
+ * @param [options.session] a session whose fingerprint and cookies the request uses, like a page request would
41
42
  */
42
43
  static find(url: string, options?: {
43
44
  signal?: AbortSignal;
44
45
  timeoutMillis?: number;
45
46
  proxyUrl?: string;
46
47
  httpClient?: BaseHttpClient;
48
+ session?: ISession;
47
49
  logger?: CrawleeLogger;
48
50
  }): Promise<RobotsTxtFile>;
49
51
  /**
@@ -38,6 +38,7 @@ export class RobotsTxtFile {
38
38
  * @param [options.signal] an AbortSignal to cancel the request
39
39
  * @param [options.timeoutMillis] timeout in milliseconds for the request
40
40
  * @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
41
+ * @param [options.session] a session whose fingerprint and cookies the request uses, like a page request would
41
42
  */
42
43
  static async find(url, options) {
43
44
  const robotsTxtFileUrl = new URL(url);
@@ -56,8 +57,9 @@ export class RobotsTxtFile {
56
57
  return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
57
58
  }
58
59
  static async #load(url, options) {
59
- const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
60
+ const { proxyUrl, logger, session, httpClient = new FetchHttpClient() } = options || {};
60
61
  const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
62
+ session,
61
63
  proxyUrl,
62
64
  timeoutMillis: options?.timeoutMillis,
63
65
  signal: options?.signal,
@@ -24,23 +24,13 @@ class SitemapTxtParser extends Transform {
24
24
  });
25
25
  }
26
26
  #processBuffer(input, finalize) {
27
- this.#buffer += input;
28
- if (finalize || this.#buffer.includes('\n')) {
29
- const parts = this.#buffer
30
- .split('\n')
31
- .map((part) => part.trim())
32
- .filter((part) => part.length > 0);
33
- if (finalize) {
34
- for (const url of parts) {
35
- this.push({ type: 'url', loc: url });
36
- }
37
- this.#buffer = '';
38
- }
39
- else if (parts.length > 0) {
40
- for (const url of parts.slice(0, -1)) {
41
- this.push({ type: 'url', loc: url });
42
- }
43
- this.#buffer = parts.at(-1);
27
+ const lines = (this.#buffer + input).split('\n');
28
+ // The text after the last line break may be a partial line, so it waits for the next chunk.
29
+ this.#buffer = finalize ? '' : lines.pop();
30
+ for (const line of lines) {
31
+ const url = line.trim();
32
+ if (url.length > 0) {
33
+ this.push({ type: 'url', loc: url });
44
34
  }
45
35
  }
46
36
  }
@@ -106,16 +96,20 @@ class SitemapXmlParser extends Transform {
106
96
  }
107
97
  }
108
98
  #onText(text) {
99
+ text = text.trim();
100
+ // Whitespace around a CDATA section arrives as separate text events.
101
+ if (text === '') {
102
+ return;
103
+ }
109
104
  if (this.#currentTag === 'loc') {
110
105
  if (this.#rootTagName === 'sitemapindex') {
111
- this.push({ type: 'sitemapUrl', url: text.trim() });
106
+ this.push({ type: 'sitemapUrl', url: text });
112
107
  }
113
108
  if (this.#rootTagName === 'urlset') {
114
109
  this.#url ??= {};
115
- this.#url.loc = text.trim();
110
+ this.#url.loc = text;
116
111
  }
117
112
  }
118
- text = text.trim();
119
113
  if (this.#currentTag === 'lastmod') {
120
114
  const lastmod = new Date(text);
121
115
  if (!Number.isNaN(lastmod.getTime())) {
@@ -150,6 +144,10 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
150
144
  if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
151
145
  return new SitemapTxtParser();
152
146
  }
147
+ // A name like `sitemap.gz` says nothing about what was compressed. Sitemaps are XML unless named otherwise.
148
+ if (mimeType?.essence === 'application/gzip') {
149
+ return SitemapXmlParser.create();
150
+ }
153
151
  throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);
154
152
  };
155
153
  while (sources.length > 0) {
@@ -241,44 +239,66 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
241
239
  // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
242
240
  // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
243
241
  let droppedUrlEntries = 0;
244
- for await (const item of items) {
245
- if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
246
- if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
247
- logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
248
- continue;
242
+ const iterator = items[Symbol.asyncIterator]();
243
+ try {
244
+ while (true) {
245
+ let next;
246
+ try {
247
+ next = await iterator.next();
249
248
  }
250
- // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
251
- // parent. Raw string sources have no parent URL, so the check is skipped.
252
- if (source.type === 'url') {
253
- const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
254
- if (!allowed) {
255
- logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
256
- continue;
249
+ catch (e) {
250
+ // Parsing failed: give up on this sitemap, keep the others.
251
+ // Raw sources already logged the error in their pipeline callback.
252
+ if (source.type !== 'raw') {
253
+ logger?.warning(`Malformed sitemap content: ${source.url} (${e})`);
257
254
  }
255
+ break;
258
256
  }
259
- sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
260
- if (emitNestedSitemaps) {
261
- yield { loc: item.url, originSitemapUrl: null };
262
- }
263
- }
264
- if (item.type === 'url') {
265
- // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
266
- if (source.type === 'url') {
267
- const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
268
- if (!allowed) {
269
- droppedUrlEntries++;
270
- logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
257
+ if (next.done)
258
+ break;
259
+ const item = next.value;
260
+ if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
261
+ if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
262
+ logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
271
263
  continue;
272
264
  }
265
+ // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
266
+ // parent. Raw string sources have no parent URL, so the check is skipped.
267
+ if (source.type === 'url') {
268
+ const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
269
+ if (!allowed) {
270
+ logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
271
+ continue;
272
+ }
273
+ }
274
+ sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
275
+ if (emitNestedSitemaps) {
276
+ yield { loc: item.url, originSitemapUrl: null };
277
+ }
278
+ }
279
+ if (item.type === 'url') {
280
+ // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
281
+ if (source.type === 'url') {
282
+ const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
283
+ if (!allowed) {
284
+ droppedUrlEntries++;
285
+ logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
286
+ continue;
287
+ }
288
+ }
289
+ yield {
290
+ ...item,
291
+ originSitemapUrl: source.type === 'url'
292
+ ? source.url
293
+ : `raw://${createHash('sha256').update(source.content).digest('base64')}`,
294
+ };
273
295
  }
274
- yield {
275
- ...item,
276
- originSitemapUrl: source.type === 'url'
277
- ? source.url
278
- : `raw://${createHash('sha256').update(source.content).digest('base64')}`,
279
- };
280
296
  }
281
297
  }
298
+ finally {
299
+ // Mirrors `for await` cleanup: destroys the stream when we stop early or the consumer bails out.
300
+ await iterator.return?.();
301
+ }
282
302
  if (droppedUrlEntries > 0 && source.type === 'url') {
283
303
  logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
284
304
  }
@@ -19,6 +19,8 @@ export declare function emailsFromText(text: string): string[];
19
19
  /**
20
20
  * The function extracts email addresses from a list of URLs.
21
21
  * Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
22
+ * A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
23
+ * such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
22
24
  * Note that the function preserves the order of emails and keep duplicates.
23
25
  * @param urls Array of URLs.
24
26
  * @return Array of emails addresses found.
@@ -30,6 +30,8 @@ export function emailsFromText(text) {
30
30
  /**
31
31
  * The function extracts email addresses from a list of URLs.
32
32
  * Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
33
+ * A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
34
+ * such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
33
35
  * Note that the function preserves the order of emails and keep duplicates.
34
36
  * @param urls Array of URLs.
35
37
  * @return Array of emails addresses found.
@@ -44,9 +46,20 @@ export function emailsFromUrls(urls) {
44
46
  continue;
45
47
  if (!EMAIL_URL_PREFIX_REGEX.test(url))
46
48
  continue;
47
- const email = url.replace(EMAIL_URL_PREFIX_REGEX, '').trim();
48
- if (EMAIL_REGEX.test(email))
49
- emails.push(email);
49
+ // Characters like `?` and `,` inside an address must be percent-encoded, so it's safe
50
+ // to split on them before decoding.
51
+ const [addresses] = url.replace(EMAIL_URL_PREFIX_REGEX, '').split('?');
52
+ for (const address of addresses.split(',')) {
53
+ let email = address.trim();
54
+ try {
55
+ email = decodeURIComponent(email).trim();
56
+ }
57
+ catch {
58
+ // Keep the raw value if it isn't valid percent-encoding
59
+ }
60
+ if (EMAIL_REGEX.test(email))
61
+ emails.push(email);
62
+ }
50
63
  }
51
64
  return emails;
52
65
  }
@@ -138,8 +151,18 @@ export function phonesFromUrls(urls) {
138
151
  continue;
139
152
  if (!PHONE_URL_PREFIX_REGEX.test(url))
140
153
  continue;
141
- const phone = url.replace(PHONE_URL_PREFIX_REGEX, '').trim();
142
- if (PHONE_REGEX.test(phone))
154
+ // Per RFC 3966, parameters such as `;ext=89` can follow the number, and `-`, `.`, `(` and `)`
155
+ // are only visual separators. They are checked without separators too, so a one-digit country
156
+ // code like in `tel:+1-555-123-4567` is accepted.
157
+ const [number] = url.replace(PHONE_URL_PREFIX_REGEX, '').split(';');
158
+ let phone = number.trim();
159
+ try {
160
+ phone = decodeURIComponent(phone).trim();
161
+ }
162
+ catch {
163
+ // Keep the raw value if it isn't valid percent-encoding
164
+ }
165
+ if (PHONE_REGEX.test(phone) || PHONE_REGEX.test(phone.replace(/[-.() ]/g, '')))
143
166
  phones.push(phone);
144
167
  }
145
168
  return phones;
@@ -54,10 +54,8 @@ export declare enum EnqueueStrategy {
54
54
  export declare const UNSUPPORTED_SCHEME_MESSAGE = "unsupported URL scheme (only http and https are allowed)";
55
55
  /**
56
56
  * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
57
- * considered here (use {@link filterUrl} for the combined scheme + strategy check).
58
- *
59
- * The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
60
- * (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
57
+ * considered here (use {@link filterUrl} for the combined scheme + strategy check). This is the single
58
+ * definition of the strategies, shared by `enqueueLinks`, sitemaps and robots.txt.
61
59
  */
62
60
  export declare function matchesEnqueueStrategy(strategy: EnqueueStrategy | `${EnqueueStrategy}`, target: URL, origin: URL): boolean;
63
61
  /**
package/internals/url.js CHANGED
@@ -71,10 +71,8 @@ function normalizeHostname(hostname) {
71
71
  }
72
72
  /**
73
73
  * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
74
- * considered here (use {@link filterUrl} for the combined scheme + strategy check).
75
- *
76
- * The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
77
- * (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
74
+ * considered here (use {@link filterUrl} for the combined scheme + strategy check). This is the single
75
+ * definition of the strategies, shared by `enqueueLinks`, sitemaps and robots.txt.
78
76
  */
79
77
  export function matchesEnqueueStrategy(strategy, target, origin) {
80
78
  switch (strategy) {
@@ -87,8 +85,8 @@ export function matchesEnqueueStrategy(strategy, target, origin) {
87
85
  if (originDomain) {
88
86
  return originDomain === getDomain(target.hostname, { mixedInputs: false });
89
87
  }
90
- // No registrable domain (e.g. an IP address), fall back to comparing origins.
91
- return target.origin === origin.origin;
88
+ // No registrable domain (e.g. an IP address or `localhost`), fall back to the same hostname.
89
+ return matchesEnqueueStrategy('same-hostname', target, origin);
92
90
  }
93
91
  case 'same-origin':
94
92
  // Compare scheme/host/port directly so a trailing-dot host is normalized.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/utils",
3
- "version": "4.0.0-rc.1",
3
+ "version": "4.0.0-rc.2",
4
4
  "description": "A set of shared utilities that can be used by crawlers",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -42,19 +42,19 @@
42
42
  "copy": "tsx ../../scripts/copy.ts"
43
43
  },
44
44
  "dependencies": {
45
- "@apify/validations": "^1.1.0",
46
- "@crawlee/http-client": "4.0.0-rc.1",
47
- "@crawlee/types": "4.0.0-rc.1",
45
+ "@apify/validations": "^1.1.4",
46
+ "@crawlee/http-client": "4.0.0-rc.2",
47
+ "@crawlee/types": "4.0.0-rc.2",
48
48
  "@types/sax": "^1.2.7",
49
- "cheerio": "^1.0.0",
50
- "domhandler": "^6.0.0",
51
- "file-type": "^21.0.0",
49
+ "cheerio": "^1.2.0",
50
+ "domhandler": "^6.0.1",
51
+ "file-type": "^21.3.4",
52
52
  "robots-parser": "^3.0.1",
53
- "sax": "^1.4.1",
54
- "tldts": "^7.0.6",
55
- "tslib": "^2.8.1",
53
+ "sax": "^1.6.1",
54
+ "tldts": "^7.4.19",
55
+ "tslib": "^2.9.1",
56
56
  "whatwg-mimetype": "^5.0.0",
57
- "zod": "^4.5.4"
57
+ "zod": "^4.6.7"
58
58
  },
59
59
  "lerna": {
60
60
  "command": {
@@ -63,5 +63,5 @@
63
63
  }
64
64
  }
65
65
  },
66
- "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
66
+ "gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
67
67
  }