@crawlee/utils 4.0.0-rc.0 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
34
34
 
35
35
  We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
36
36
 
37
- > Crawlee requires **Node.js 16 or higher**.
37
+ > Crawlee requires **Node.js 22.13 or higher**.
38
38
 
39
39
  ### With Crawlee CLI
40
40
 
package/index.d.ts CHANGED
@@ -2,9 +2,11 @@ export { htmlToText } from './internals/cheerio.js';
2
2
  export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
3
3
  export { EnqueueStrategy } from './internals/url.js';
4
4
  export type { DownloadListOfUrlsOptions, ExtractUrlsOptions } from './internals/extract-urls.js';
5
- export { sleep, expandShadowRoots } from './internals/general.js';
5
+ export { sleep } from './internals/general.js';
6
6
  export * as social from './internals/social.js';
7
- export * from './internals/open_graph_parser.js';
7
+ export * from './internals/extract-microdata.js';
8
+ export * from './internals/open-graph-parser.js';
8
9
  export * from './internals/robots.js';
9
- export * from './internals/sitemap.js';
10
- export * from './internals/validation.js';
10
+ export { discoverValidSitemaps, Sitemap } from './internals/sitemap.js';
11
+ export type { ParseSitemapOptions } from './internals/sitemap.js';
12
+ export { ArgumentValidationError } from './internals/validation.js';
package/index.js CHANGED
@@ -1,9 +1,10 @@
1
1
  export { htmlToText } from './internals/cheerio.js';
2
2
  export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
3
3
  export { EnqueueStrategy } from './internals/url.js';
4
- export { sleep, expandShadowRoots } from './internals/general.js';
4
+ export { sleep } from './internals/general.js';
5
5
  export * as social from './internals/social.js';
6
- export * from './internals/open_graph_parser.js';
6
+ export * from './internals/extract-microdata.js';
7
+ export * from './internals/open-graph-parser.js';
7
8
  export * from './internals/robots.js';
8
- export * from './internals/sitemap.js';
9
- export * from './internals/validation.js';
9
+ export { discoverValidSitemaps, Sitemap } from './internals/sitemap.js';
10
+ export { ArgumentValidationError } from './internals/validation.js';
package/internal.d.ts CHANGED
@@ -1,9 +1,12 @@
1
1
  export * from './internals/blocked.js';
2
- export type { CheerioRoot, CheerioAPI, Cheerio, Element } from './internals/cheerio.js';
2
+ export type { CheerioAPI, Cheerio, Element } from './internals/cheerio.js';
3
3
  export { extractUrlsFromCheerio } from './internals/cheerio.js';
4
4
  export { tryAbsoluteURL } from './internals/extract-urls.js';
5
- export { URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
5
+ export { expandShadowRoots, URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
6
+ export { processHttpRequestOptions } from './internals/http-request.js';
6
7
  export * from './internals/iterables.js';
8
+ export { MutationJournal } from './internals/mutation-journal.js';
7
9
  export * from './internals/url.js';
8
10
  export * from './internals/validation.js';
9
11
  export * as schemas from './internals/schemas.js';
12
+ export { parseSitemap, type SitemapUrl } from './internals/sitemap.js';
package/internal.js CHANGED
@@ -1,8 +1,11 @@
1
1
  export * from './internals/blocked.js';
2
2
  export { extractUrlsFromCheerio } from './internals/cheerio.js';
3
3
  export { tryAbsoluteURL } from './internals/extract-urls.js';
4
- export { URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
4
+ export { expandShadowRoots, URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
5
+ export { processHttpRequestOptions } from './internals/http-request.js';
5
6
  export * from './internals/iterables.js';
7
+ export { MutationJournal } from './internals/mutation-journal.js';
6
8
  export * from './internals/url.js';
7
9
  export * from './internals/validation.js';
8
10
  export * as schemas from './internals/schemas.js';
11
+ export { parseSitemap } from './internals/sitemap.js';
@@ -1,4 +1,11 @@
1
1
  export declare const CLOUDFLARE_RETRY_CSS_SELECTORS: string[];
2
+ /**
3
+ * The AWS WAF interstitials: the JS challenge, served with a `202` status, and the CAPTCHA it escalates to, served with a `405`.
4
+ * Neither status is a blocked one. Matching the container as a direct child of `<body>` keeps regular pages
5
+ * that load the AWS WAF integration scripts, or embed the CAPTCHA into their own markup, from matching.
6
+ * The `:has()` argument must not start with `>`, Playwright's selector engine never matches it on the root element.
7
+ */
8
+ export declare const AWS_WAF_RETRY_CSS_SELECTORS: string[];
2
9
  /**
3
10
  * CSS selectors for elements that should trigger a retry, as the crawler is likely getting blocked.
4
11
  */
@@ -1,9 +1,20 @@
1
1
  export const CLOUDFLARE_RETRY_CSS_SELECTORS = ['#turnstile-wrapper iframe[src^="https://challenges.cloudflare.com"]'];
2
+ /**
3
+ * The AWS WAF interstitials: the JS challenge, served with a `202` status, and the CAPTCHA it escalates to, served with a `405`.
4
+ * Neither status is a blocked one. Matching the container as a direct child of `<body>` keeps regular pages
5
+ * that load the AWS WAF integration scripts, or embed the CAPTCHA into their own markup, from matching.
6
+ * The `:has()` argument must not start with `>`, Playwright's selector engine never matches it on the root element.
7
+ */
8
+ export const AWS_WAF_RETRY_CSS_SELECTORS = [
9
+ 'html:has(head > script[src*=".awswaf.com/"][src$="/challenge.js"]) > body > #challenge-container',
10
+ 'html:has(head > script[src*=".awswaf.com/"][src$="/captcha.js"]) > body > #captcha-container',
11
+ ];
2
12
  /**
3
13
  * CSS selectors for elements that should trigger a retry, as the crawler is likely getting blocked.
4
14
  */
5
15
  export const RETRY_CSS_SELECTORS = [
6
16
  ...CLOUDFLARE_RETRY_CSS_SELECTORS,
17
+ ...AWS_WAF_RETRY_CSS_SELECTORS,
7
18
  'div#infoDiv0 a[href*="//www.google.com/policies/terms/"]',
8
19
  'iframe[src*="_Incapsula_Resource"]',
9
20
  ];
@@ -1,5 +1,4 @@
1
1
  import type { CheerioAPI } from 'cheerio';
2
- export type CheerioRoot = CheerioAPI;
3
2
  export type { CheerioAPI, Cheerio } from 'cheerio';
4
3
  export type { Element } from 'domhandler';
5
4
  /**
@@ -29,7 +28,7 @@ export type { Element } from 'domhandler';
29
28
  * @param htmlOrCheerioElement HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function.
30
29
  * @return Plain text
31
30
  */
32
- export declare function htmlToText(htmlOrCheerioElement: string | CheerioRoot): Promise<string>;
31
+ export declare function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Promise<string>;
33
32
  /**
34
33
  * Extracts URLs from a given Cheerio object.
35
34
  *
@@ -1,7 +1,7 @@
1
1
  import { tryAbsoluteURL } from './extract-urls.js';
2
2
  // NOTE: We are skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.
3
3
  const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;
4
- const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|div|fieldset|form|table|tr|select|option)$/i;
4
+ const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|dt|dd|div|fieldset|legend|form|table|caption|tr|select|option|hr|article|aside|section|nav|header|footer|main|hgroup|figure|figcaption|details|summary|dialog|menu)$/i;
5
5
  /**
6
6
  * The function converts a HTML document to a plain text.
7
7
  *
@@ -30,52 +30,63 @@ const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|
30
30
  * @return Plain text
31
31
  */
32
32
  export async function htmlToText(htmlOrCheerioElement) {
33
- const { load } = await import('cheerio');
33
+ const { load } = await import('cheerio/slim');
34
34
  if (!htmlOrCheerioElement)
35
35
  return '';
36
36
  const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : load(htmlOrCheerioElement);
37
- let text = '';
38
- const process = (elems) => {
37
+ // Never read the accumulated text mid-loop: each read flattens V8's cons-string, making the traversal quadratic
38
+ const parts = [];
39
+ let last = '';
40
+ const push = (part) => {
41
+ if (!part)
42
+ return;
43
+ parts.push(part);
44
+ last = part[part.length - 1];
45
+ };
46
+ const process = (elems, inPre = false) => {
39
47
  const len = elems ? elems.length : 0;
40
48
  for (let i = 0; i < len; i++) {
41
49
  const elem = elems[i];
42
50
  if (elem.type === 'text') {
43
- // Compress spaces, unless we're inside <pre> element
51
+ // Compress spaces, unless we're inside <pre> element (including nested elements like <pre><code>)
44
52
  let compr;
45
- if (elem.parent?.tagName === 'pre')
46
- compr = elem.data;
53
+ if (inPre) {
54
+ // Strip single newline right after `<pre>` that htmlparser2 keeps
55
+ const isFirstInPre = elem.parent?.tagName === 'pre' && elem.parent.children?.[0] === elem;
56
+ compr = isFirstInPre ? elem.data.replace(/^\r?\n/, '') : elem.data;
57
+ }
47
58
  else
48
59
  compr = elem.data.replace(/\s+/g, ' ');
49
60
  // If text is empty or ends with a whitespace, don't add the leading whitespace
50
- if (compr.startsWith(' ') && /(^|\s)$/.test(text))
61
+ if (!inPre && compr.startsWith(' ') && (last === '' || /\s/.test(last)))
51
62
  compr = compr.substring(1);
52
- text += compr;
63
+ push(compr);
53
64
  }
54
65
  else if (elem.type === 'comment' || SKIP_TAGS_REGEX.test(elem.tagName)) {
55
66
  // Skip comments and special elements
56
67
  }
57
68
  else if (elem.tagName === 'br') {
58
- text += '\n';
69
+ push('\n');
59
70
  }
60
- else if (elem.tagName === 'td') {
61
- process(elem.children);
62
- text += '\t';
71
+ else if (elem.tagName === 'td' || elem.tagName === 'th') {
72
+ process(elem.children, inPre);
73
+ push('\t');
63
74
  }
64
75
  else {
65
76
  // Block elements must be surrounded by newlines (unless beginning of text)
66
77
  const isBlockTag = BLOCK_TAGS_REGEX.test(elem.tagName);
67
- if (isBlockTag && !/(^|\n)$/.test(text))
68
- text += '\n';
69
- process(elem.children);
70
- if (isBlockTag && !text.endsWith('\n'))
71
- text += '\n';
78
+ if (isBlockTag && last !== '' && last !== '\n')
79
+ push('\n');
80
+ process(elem.children, inPre || elem.tagName === 'pre');
81
+ if (isBlockTag && last !== '\n')
82
+ push('\n');
72
83
  }
73
84
  }
74
85
  };
75
86
  // If HTML document has body, only convert that, otherwise convert the entire HTML
76
87
  const $body = $('body');
77
88
  process($body.length > 0 ? $body : $.root());
78
- return text.trim();
89
+ return parts.join('').trim();
79
90
  }
80
91
  /**
81
92
  * Extracts URLs from a given Cheerio object.
@@ -0,0 +1,24 @@
1
+ import type { CheerioAPI } from 'cheerio';
2
+ /** The value of a microdata property: either text or a nested item. */
3
+ export type MicrodataValue = string | MicrodataItem;
4
+ /** A single schema.org item extracted from a document. */
5
+ export interface MicrodataItem {
6
+ /** Tokens of the item's `itemtype` attribute. */
7
+ type?: string[];
8
+ /** The item's `itemid` attribute. */
9
+ id?: string;
10
+ /** Values keyed by `itemprop` name, an array where the property repeats. */
11
+ properties: Record<string, MicrodataValue | MicrodataValue[]>;
12
+ }
13
+ /**
14
+ * Easily parse all schema.org microdata from a page with just a `CheerioAPI` object or raw HTML,
15
+ * following the [microdata processing model](https://html.spec.whatwg.org/multipage/microdata.html#microdata).
16
+ *
17
+ * Text values are trimmed and their inner whitespace collapsed. URL-valued attributes are returned
18
+ * verbatim rather than resolved against the document's base URL.
19
+ *
20
+ * @param htmlOrCheerioElement A `CheerioAPI` object, or a string of raw HTML.
21
+ * @returns The document's top-level items. Nested items are the property values of their parent.
22
+ */
23
+ export declare function extractMicrodata(raw: string): Promise<MicrodataItem[]>;
24
+ export declare function extractMicrodata($: CheerioAPI): Promise<MicrodataItem[]>;
@@ -0,0 +1,122 @@
1
+ import { isTag } from 'domhandler';
2
+ export async function extractMicrodata(htmlOrCheerioElement) {
3
+ // Dynamic so that importing `@crawlee/utils` does not pull in cheerio - see #3836.
4
+ const { load } = await import('cheerio/slim');
5
+ const $ = typeof htmlOrCheerioElement === 'string' ? load(htmlOrCheerioElement) : htmlOrCheerioElement;
6
+ const context = { $ };
7
+ return $('[itemscope]')
8
+ .toArray()
9
+ .filter((element) => !('itemprop' in element.attribs))
10
+ .map((element) => parseItem(context, element, new Set()));
11
+ }
12
+ function parseItem(context, element, ancestors) {
13
+ const item = { properties: {} };
14
+ const type = uniqueTokens(element.attribs.itemtype);
15
+ const id = element.attribs.itemid;
16
+ if (type.length > 0) {
17
+ item.type = type;
18
+ }
19
+ if (id) {
20
+ item.id = id.trim();
21
+ }
22
+ ancestors.add(element);
23
+ for (const propertyElement of collectPropertyElements(context, element)) {
24
+ const value = getPropertyValue(context, propertyElement, ancestors);
25
+ for (const name of uniqueTokens(propertyElement.attribs.itemprop)) {
26
+ addProperty(item.properties, name, value);
27
+ }
28
+ }
29
+ ancestors.delete(element);
30
+ return item;
31
+ }
32
+ function collectPropertyElements(context, scope) {
33
+ const elements = [];
34
+ // An `itemref` can point at an element already reached from the item's own subtree, or at one that
35
+ // contains another referenced element. The spec visits every element once, so each property counts once.
36
+ const visited = new Set();
37
+ collectFromNodes(scope.children, elements, visited);
38
+ for (const id of uniqueTokens(scope.attribs.itemref)) {
39
+ const referenced = (context.idIndex ??= indexIds(context.$)).get(id);
40
+ if (referenced) {
41
+ collectFromNodes([referenced], elements, visited);
42
+ }
43
+ }
44
+ return elements;
45
+ }
46
+ function collectFromNodes(nodes, elements, visited) {
47
+ for (const node of nodes) {
48
+ if (!isTag(node) || visited.has(node)) {
49
+ continue;
50
+ }
51
+ visited.add(node);
52
+ if ('itemprop' in node.attribs) {
53
+ elements.push(node);
54
+ }
55
+ // A nested item owns everything below it, so its subtree is not part of the enclosing item.
56
+ if (!('itemscope' in node.attribs)) {
57
+ collectFromNodes(node.children, elements, visited);
58
+ }
59
+ }
60
+ }
61
+ function indexIds($) {
62
+ const index = new Map();
63
+ for (const element of $('[id]').toArray()) {
64
+ // Duplicate ids are invalid HTML; `getElementById` resolves them to the first element.
65
+ if (!index.has(element.attribs.id)) {
66
+ index.set(element.attribs.id, element);
67
+ }
68
+ }
69
+ return index;
70
+ }
71
+ function getPropertyValue(context, element, ancestors) {
72
+ if ('itemscope' in element.attribs) {
73
+ // `itemref` can point back at an enclosing item, which the spec treats as an error.
74
+ return ancestors.has(element) ? { properties: {} } : parseItem(context, element, ancestors);
75
+ }
76
+ const { attribs } = element;
77
+ switch (element.tagName.toLowerCase()) {
78
+ case 'meta':
79
+ return attribs.content ?? '';
80
+ case 'audio':
81
+ case 'embed':
82
+ case 'iframe':
83
+ case 'img':
84
+ case 'source':
85
+ case 'track':
86
+ case 'video':
87
+ return attribs.src ?? '';
88
+ case 'a':
89
+ case 'area':
90
+ case 'link':
91
+ return attribs.href ?? '';
92
+ case 'object':
93
+ return attribs.data ?? '';
94
+ case 'data':
95
+ case 'meter':
96
+ return attribs.value ?? '';
97
+ case 'time':
98
+ return attribs.datetime ?? context.$(element).text().replace(/\s+/g, ' ').trim();
99
+ default:
100
+ return context.$(element).text().replace(/\s+/g, ' ').trim();
101
+ }
102
+ }
103
+ function addProperty(properties, name, value) {
104
+ const existing = properties[name];
105
+ if (existing === undefined) {
106
+ properties[name] = value;
107
+ }
108
+ else if (Array.isArray(existing)) {
109
+ existing.push(value);
110
+ }
111
+ else {
112
+ properties[name] = [existing, value];
113
+ }
114
+ }
115
+ /** `itemtype`, `itemprop` and `itemref` are all unordered sets of unique space-separated tokens. */
116
+ function uniqueTokens(value) {
117
+ if (!value) {
118
+ return [];
119
+ }
120
+ const tokens = value.split(/\s+/).filter(Boolean);
121
+ return tokens.length > 1 ? [...new Set(tokens)] : tokens;
122
+ }
@@ -20,13 +20,7 @@ const extractUrlsOptionsSchema = z.strictObject({
20
20
  */
21
21
  export async function downloadListOfUrls(options) {
22
22
  const { url, encoding, urlRegExp, proxyUrl, httpClient } = parseArgument(options, downloadListOfUrlsOptionsSchema);
23
- // Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one.
24
- const match = /^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/.exec(url);
25
- let fixedUrl = url;
26
- if (match) {
27
- fixedUrl = `${match[1]}/gviz/tq?tqx=out:csv`;
28
- }
29
- const response = await httpClient.sendRequest(new Request(fixedUrl, { method: 'GET' }), {
23
+ const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
30
24
  proxyUrl,
31
25
  });
32
26
  const string = new TextDecoder(encoding).decode(new Uint8Array(await response.arrayBuffer()));
@@ -0,0 +1,5 @@
1
+ import type { HttpRequest, HttpRequestOptions } from '@crawlee/types';
2
+ /**
3
+ * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
4
+ */
5
+ export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
@@ -0,0 +1,36 @@
1
+ import { Readable } from 'node:stream';
2
+ import { applySearchParams } from './url.js';
3
+ /**
4
+ * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
5
+ */
6
+ export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
7
+ const url = new URL(request.url);
8
+ const headers = new Headers(request.headers);
9
+ applySearchParams(url, searchParams);
10
+ if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
11
+ throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
12
+ }
13
+ const body = (() => {
14
+ if (form !== undefined) {
15
+ return Readable.from(new URLSearchParams(form).toString());
16
+ }
17
+ if (json !== undefined) {
18
+ return Readable.from(JSON.stringify(json));
19
+ }
20
+ if (request.body !== undefined) {
21
+ return Readable.from(request.body);
22
+ }
23
+ return undefined;
24
+ })();
25
+ if (form !== undefined && !headers.has('content-type')) {
26
+ headers.set('content-type', 'application/x-www-form-urlencoded');
27
+ }
28
+ if (json !== undefined && !headers.has('content-type')) {
29
+ headers.set('content-type', 'application/json');
30
+ }
31
+ if (username !== undefined || password !== undefined) {
32
+ const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
33
+ headers.set('authorization', `Basic ${encodedAuth}`);
34
+ }
35
+ return { ...request, body, url, headers };
36
+ }
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Records property writes made through the proxies it hands out, applying them to the real objects as
3
+ * they happen, so that a losing attempt's writes can be undone afterwards - on the objects themselves
4
+ * ({@link MutationJournal.revert|`revert()`}), or on a copy of them
5
+ * ({@link MutationJournal.undoOnto|`undoOnto()`}) to reconstruct how they looked before the attempt.
6
+ *
7
+ * An undo on the live object only touches a property that still holds the value the attempt wrote - a
8
+ * value written by someone else in the meantime is left alone and reported via `onConflict`. Writes that
9
+ * bypass the proxy are invisible until they trip that guard.
10
+ */
11
+ export declare class MutationJournal {
12
+ #private;
13
+ track<T extends object>(target: T, path?: PropertyKey[]): T;
14
+ /** Stops recording. Proxies keep writing through. */
15
+ close(): void;
16
+ /** Undoes the recorded writes, newest first, and stops recording. */
17
+ revert({ onConflict }?: {
18
+ onConflict?: (key: PropertyKey) => void;
19
+ }): void;
20
+ /**
21
+ * Applies the inverse of the recorded writes onto `root`, a JSON copy of the tracked object, and returns
22
+ * it. The journal is left as it is. Restored values are JSON-cloned too, so the copy shares nothing with
23
+ * the original and non-JSON values drop out the same way on both.
24
+ *
25
+ * Entries are located by the path under which their object was *first* reached, so an object reachable
26
+ * through two paths is undone under the first one only.
27
+ */
28
+ undoOnto<T extends object>(root: T): T;
29
+ }
@@ -0,0 +1,173 @@
1
+ /**
2
+ * Only plain objects and arrays are wrapped when reached through a tracked object. Built-ins such as `Map`
3
+ * or `Set` reject a proxy receiver in their methods, and the tracked objects are JSON-shaped by contract
4
+ * anyway - such values are handed out as they are, and their mutations are not recorded.
5
+ */
6
+ const isTrackable = (value) => Array.isArray(value) ||
7
+ (typeof value === 'object' &&
8
+ value !== null &&
9
+ (Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null));
10
+ /** Every proxy any journal has handed out, keyed by the proxy. Shared, as a value read through one attempt's
11
+ * proxy may be written back through another's. */
12
+ const proxyTargets = new WeakMap();
13
+ /**
14
+ * Replaces journal proxies in `value` with the objects behind them, in place for nested ones. A value read
15
+ * through a proxy and written back (`state.x = state.x ?? {}`) would otherwise store the proxy itself, and
16
+ * every later attempt would wrap it once more.
17
+ */
18
+ function unwrap(value, seen = new Set()) {
19
+ const target = typeof value === 'object' && value !== null ? proxyTargets.get(value) : undefined;
20
+ if (target)
21
+ return target;
22
+ if (!isTrackable(value) || seen.has(value))
23
+ return value;
24
+ seen.add(value);
25
+ for (const key of Object.keys(value)) {
26
+ const inner = value[key];
27
+ const unwrapped = unwrap(inner, seen);
28
+ if (unwrapped !== inner)
29
+ value[key] = unwrapped;
30
+ }
31
+ return value;
32
+ }
33
+ /**
34
+ * Records property writes made through the proxies it hands out, applying them to the real objects as
35
+ * they happen, so that a losing attempt's writes can be undone afterwards - on the objects themselves
36
+ * ({@link MutationJournal.revert|`revert()`}), or on a copy of them
37
+ * ({@link MutationJournal.undoOnto|`undoOnto()`}) to reconstruct how they looked before the attempt.
38
+ *
39
+ * An undo on the live object only touches a property that still holds the value the attempt wrote - a
40
+ * value written by someone else in the meantime is left alone and reported via `onConflict`. Writes that
41
+ * bypass the proxy are invisible until they trip that guard.
42
+ */
43
+ export class MutationJournal {
44
+ #entries = [];
45
+ #proxies = new WeakMap();
46
+ #paths = new WeakMap();
47
+ #closed = false;
48
+ track(target, path = []) {
49
+ const cached = this.#proxies.get(target);
50
+ if (cached)
51
+ return cached;
52
+ this.#paths.set(target, path);
53
+ const entry = (key, fields) => this.#entries.push({ target, path, key, ...fields });
54
+ // Accessors run with `this` as the real object, not the proxy: class accessors may touch `#private`
55
+ // members, which throw on a proxy receiver. An accessor write is therefore journaled as its own
56
+ // entry and undone through the same setter, rather than through whatever it writes underneath.
57
+ const proxy = new Proxy(target, {
58
+ get: (object, key) => {
59
+ const value = Reflect.get(object, key);
60
+ return isTrackable(value) ? this.track(value, [...path, key]) : value;
61
+ },
62
+ set: (object, key, rawNext) => {
63
+ const next = unwrap(rawNext);
64
+ const had = Reflect.has(object, key);
65
+ const prev = Reflect.get(object, key);
66
+ this.#recordArrayTruncation(object, key, next);
67
+ if (!Reflect.set(object, key, next))
68
+ return false;
69
+ // An array's `length` is implied by its elements and never undone on its own.
70
+ if (!this.#closed && !(Array.isArray(object) && key === 'length')) {
71
+ entry(key, { had, prev, next, deleted: false });
72
+ }
73
+ return true;
74
+ },
75
+ deleteProperty: (object, key) => {
76
+ const had = Reflect.has(object, key);
77
+ const prev = Reflect.get(object, key);
78
+ if (!Reflect.deleteProperty(object, key))
79
+ return false;
80
+ if (!this.#closed && had)
81
+ entry(key, { had, prev, next: undefined, deleted: true });
82
+ return true;
83
+ },
84
+ });
85
+ this.#proxies.set(target, proxy);
86
+ proxyTargets.set(proxy, target);
87
+ return proxy;
88
+ }
89
+ /**
90
+ * Shrinking an array's `length` drops elements without a trap firing for them. They are recorded as
91
+ * deletions, from the highest index down, so the reverse replay reinserts them from the lowest up.
92
+ */
93
+ #recordArrayTruncation(object, key, next) {
94
+ if (this.#closed || !Array.isArray(object) || key !== 'length')
95
+ return;
96
+ const path = this.#paths.get(object);
97
+ for (let i = object.length - 1; i >= next; i--) {
98
+ if (i in object) {
99
+ this.#entries.push({
100
+ target: object,
101
+ path,
102
+ key: String(i),
103
+ had: true,
104
+ prev: object[i],
105
+ next: undefined,
106
+ deleted: true,
107
+ });
108
+ }
109
+ }
110
+ }
111
+ /**
112
+ * Restores one entry on `target`. Array elements are handled positionally - an added element is removed
113
+ * and a removed one reinserted, shifting the rest - so the result stays something a sequence of `push`
114
+ * and `splice` calls could have produced, even next to elements another writer added meanwhile.
115
+ */
116
+ static #undo(target, { key, had, deleted }, value) {
117
+ if (Array.isArray(target) && typeof key === 'string' && Number.isInteger(Number(key))) {
118
+ if (deleted) {
119
+ target.splice(Number(key), 0, value);
120
+ }
121
+ else if (had) {
122
+ target[Number(key)] = value;
123
+ }
124
+ else {
125
+ target.splice(Number(key), 1);
126
+ }
127
+ return;
128
+ }
129
+ if (had) {
130
+ Reflect.set(target, key, value);
131
+ }
132
+ else {
133
+ Reflect.deleteProperty(target, key);
134
+ }
135
+ }
136
+ /** Stops recording. Proxies keep writing through. */
137
+ close() {
138
+ this.#closed = true;
139
+ }
140
+ /** Undoes the recorded writes, newest first, and stops recording. */
141
+ revert({ onConflict } = {}) {
142
+ this.close();
143
+ for (let i = this.#entries.length - 1; i >= 0; i--) {
144
+ const entry = this.#entries[i];
145
+ const { target, key, next, deleted } = entry;
146
+ const untouched = deleted ? !Reflect.has(target, key) : Reflect.get(target, key) === next;
147
+ if (!untouched) {
148
+ onConflict?.(key);
149
+ continue;
150
+ }
151
+ MutationJournal.#undo(target, entry, entry.prev);
152
+ }
153
+ this.#entries.length = 0;
154
+ }
155
+ /**
156
+ * Applies the inverse of the recorded writes onto `root`, a JSON copy of the tracked object, and returns
157
+ * it. The journal is left as it is. Restored values are JSON-cloned too, so the copy shares nothing with
158
+ * the original and non-JSON values drop out the same way on both.
159
+ *
160
+ * Entries are located by the path under which their object was *first* reached, so an object reachable
161
+ * through two paths is undone under the first one only.
162
+ */
163
+ undoOnto(root) {
164
+ for (let i = this.#entries.length - 1; i >= 0; i--) {
165
+ const entry = this.#entries[i];
166
+ const target = entry.path.reduce((object, step) => object?.[step], root);
167
+ if (typeof target !== 'object' || target === null)
168
+ continue;
169
+ MutationJournal.#undo(target, entry, isTrackable(entry.prev) ? JSON.parse(JSON.stringify(entry.prev)) : entry.prev);
170
+ }
171
+ return root;
172
+ }
173
+ }
@@ -366,7 +366,7 @@ const parseOpenGraphProperty = (property, $) => {
366
366
  };
367
367
  };
368
368
  export async function parseOpenGraph(item, additionalProperties) {
369
- const { load } = await import('cheerio');
369
+ const { load } = await import('cheerio/slim');
370
370
  const $ = typeof item === 'string' ? load(item) : item;
371
371
  return [...(additionalProperties || []), ...OPEN_GRAPH_PROPERTIES].reduce((acc, curr) => {
372
372
  return {