@crawlee/utils 4.0.0-beta.26 → 4.0.0-beta.260

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +10 -17
  3. package/index.js +8 -16
  4. package/internal.d.ts +12 -0
  5. package/internal.js +11 -0
  6. package/internals/blocked.d.ts +0 -1
  7. package/internals/blocked.js +0 -1
  8. package/internals/cheerio.d.ts +1 -3
  9. package/internals/cheerio.js +25 -16
  10. package/internals/extract-microdata.d.ts +24 -0
  11. package/internals/extract-microdata.js +118 -0
  12. package/internals/extract-urls.d.ts +1 -2
  13. package/internals/extract-urls.js +18 -17
  14. package/internals/general.d.ts +0 -25
  15. package/internals/general.js +2 -110
  16. package/internals/http-request.d.ts +5 -0
  17. package/internals/http-request.js +36 -0
  18. package/internals/iterables.d.ts +0 -80
  19. package/internals/iterables.js +0 -126
  20. package/internals/mutation-journal.d.ts +29 -0
  21. package/internals/mutation-journal.js +173 -0
  22. package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
  23. package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
  24. package/internals/robots.d.ts +35 -15
  25. package/internals/robots.js +66 -30
  26. package/internals/schemas.d.ts +114 -0
  27. package/internals/schemas.js +115 -0
  28. package/internals/sitemap.d.ts +44 -3
  29. package/internals/sitemap.js +173 -97
  30. package/internals/social.d.ts +4 -3
  31. package/internals/social.js +36 -11
  32. package/internals/url.d.ts +67 -1
  33. package/internals/url.js +118 -2
  34. package/internals/validation.d.ts +9 -0
  35. package/internals/validation.js +16 -0
  36. package/package.json +16 -15
  37. package/index.d.ts.map +0 -1
  38. package/index.js.map +0 -1
  39. package/internals/blocked.d.ts.map +0 -1
  40. package/internals/blocked.js.map +0 -1
  41. package/internals/cheerio.d.ts.map +0 -1
  42. package/internals/cheerio.js.map +0 -1
  43. package/internals/chunk.d.ts +0 -2
  44. package/internals/chunk.d.ts.map +0 -1
  45. package/internals/chunk.js +0 -40
  46. package/internals/chunk.js.map +0 -1
  47. package/internals/debug.d.ts +0 -32
  48. package/internals/debug.d.ts.map +0 -1
  49. package/internals/debug.js +0 -41
  50. package/internals/debug.js.map +0 -1
  51. package/internals/extract-urls.d.ts.map +0 -1
  52. package/internals/extract-urls.js.map +0 -1
  53. package/internals/general.d.ts.map +0 -1
  54. package/internals/general.js.map +0 -1
  55. package/internals/iterables.d.ts.map +0 -1
  56. package/internals/iterables.js.map +0 -1
  57. package/internals/open_graph_parser.d.ts.map +0 -1
  58. package/internals/open_graph_parser.js.map +0 -1
  59. package/internals/robots.d.ts.map +0 -1
  60. package/internals/robots.js.map +0 -1
  61. package/internals/sitemap.d.ts.map +0 -1
  62. package/internals/sitemap.js.map +0 -1
  63. package/internals/social.d.ts.map +0 -1
  64. package/internals/social.js.map +0 -1
  65. package/internals/system-info/cpu-info.d.ts +0 -64
  66. package/internals/system-info/cpu-info.d.ts.map +0 -1
  67. package/internals/system-info/cpu-info.js +0 -211
  68. package/internals/system-info/cpu-info.js.map +0 -1
  69. package/internals/system-info/memory-info.d.ts +0 -28
  70. package/internals/system-info/memory-info.d.ts.map +0 -1
  71. package/internals/system-info/memory-info.js +0 -118
  72. package/internals/system-info/memory-info.js.map +0 -1
  73. package/internals/system-info/ps-tree.d.ts +0 -18
  74. package/internals/system-info/ps-tree.d.ts.map +0 -1
  75. package/internals/system-info/ps-tree.js +0 -145
  76. package/internals/system-info/ps-tree.js.map +0 -1
  77. package/internals/typedefs.d.ts +0 -5
  78. package/internals/typedefs.d.ts.map +0 -1
  79. package/internals/typedefs.js +0 -9
  80. package/internals/typedefs.js.map +0 -1
  81. package/internals/url.d.ts.map +0 -1
  82. package/internals/url.js.map +0 -1
@@ -1,108 +1,14 @@
1
- import fs from 'node:fs/promises';
2
1
  import { setTimeout } from 'node:timers/promises';
3
2
  /**
4
3
  * Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
5
4
  * and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
6
5
  */
7
- export const URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()]*)?/giu;
6
+ export const URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()'*]*)?/giu;
8
7
  /**
9
8
  * Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
10
9
  * Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
11
10
  */
12
- export const URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()]*)?/giu;
13
- let isDockerPromiseCache;
14
- async function createIsDockerPromise() {
15
- const promise1 = fs
16
- .stat('/.dockerenv')
17
- .then(() => true)
18
- .catch(() => false);
19
- const promise2 = fs
20
- .readFile('/proc/self/cgroup', 'utf8')
21
- .then((content) => content.includes('docker'))
22
- .catch(() => false);
23
- const [result1, result2] = await Promise.all([promise1, promise2]);
24
- return result1 || result2;
25
- }
26
- /**
27
- * Returns a `Promise` that resolves to true if the code is running in a Docker container.
28
- */
29
- export async function isDocker(forceReset) {
30
- // Parameter forceReset is just internal for unit tests.
31
- if (!isDockerPromiseCache || forceReset)
32
- isDockerPromiseCache = createIsDockerPromise();
33
- return isDockerPromiseCache;
34
- }
35
- let isContainerizedResult;
36
- /**
37
- * Detects if crawlee is running in a containerized environment.
38
- */
39
- export async function isContainerized() {
40
- // Value is very unlikley to change. Cache the result after the first execution.
41
- if (isContainerizedResult !== undefined) {
42
- return isContainerizedResult;
43
- }
44
- // return false if running in aws lambda
45
- if (isLambda()) {
46
- isContainerizedResult = false;
47
- return isContainerizedResult;
48
- }
49
- const dockerenvCheck = fs
50
- .stat('/.dockerenv')
51
- .then(() => true)
52
- .catch(() => false);
53
- const cgroupCheck = fs
54
- .readFile('/proc/self/cgroup', 'utf8')
55
- .then((content) => content.includes('docker'))
56
- .catch(() => false);
57
- const [dockerenvResult, cgroupResult] = await Promise.all([dockerenvCheck, cgroupCheck]);
58
- isContainerizedResult = dockerenvResult || cgroupResult || !!process.env.KUBERNETES_SERVICE_HOST;
59
- return isContainerizedResult;
60
- }
61
- export function isLambda() {
62
- return !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
63
- }
64
- let _cgroupsVersion;
65
- /**
66
- * gets the cgroup version by checking for a file at /sys/fs/cgroup/memory
67
- * @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.
68
- */
69
- export async function getCgroupsVersion(forceReset) {
70
- // Parameter forceReset is just internal for unit tests.
71
- if (_cgroupsVersion !== undefined && !forceReset) {
72
- return _cgroupsVersion;
73
- }
74
- try {
75
- // If this directory does not exists, cgroups are not available
76
- await fs.access('/sys/fs/cgroup/');
77
- }
78
- catch (e) {
79
- _cgroupsVersion = null;
80
- return null;
81
- }
82
- _cgroupsVersion = 'V1';
83
- try {
84
- // If this directory does not exists, assume the container is using cgroups V2
85
- await fs.access('/sys/fs/cgroup/memory/');
86
- }
87
- catch (e) {
88
- _cgroupsVersion = 'V2';
89
- }
90
- return _cgroupsVersion;
91
- }
92
- /**
93
- * Computes a weighted average of an array of numbers, complemented by an array of weights.
94
- * @ignore
95
- */
96
- export function weightedAvg(arrValues, arrWeights) {
97
- const result = arrValues
98
- .map((value, i) => {
99
- const weight = arrWeights[i];
100
- const sum = value * weight;
101
- return [sum, weight];
102
- })
103
- .reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);
104
- return result[0] / result[1];
105
- }
11
+ export const URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;
106
12
  /**
107
13
  * Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
108
14
  * in your code, e.g. to prevent overloading of target website or to avoid bot detection.
@@ -122,19 +28,6 @@ export function weightedAvg(arrValues, arrWeights) {
122
28
  export async function sleep(millis) {
123
29
  return setTimeout(millis ?? undefined);
124
30
  }
125
- /**
126
- * Converts SNAKE_CASE to camelCase.
127
- * @ignore
128
- */
129
- export function snakeCaseToCamelCase(snakeCaseStr) {
130
- return snakeCaseStr
131
- .toLowerCase()
132
- .split('_')
133
- .map((part, index) => {
134
- return index > 0 ? part.charAt(0).toUpperCase() + part.slice(1) : part;
135
- })
136
- .join('');
137
- }
138
31
  /**
139
32
  * Traverses DOM and expands shadow-root elements (created by custom components).
140
33
  * @ignore
@@ -164,4 +57,3 @@ export function expandShadowRoots(document) {
164
57
  replaceShadowDomsWithHtml(document.body);
165
58
  return document.documentElement.outerHTML;
166
59
  }
167
- //# sourceMappingURL=general.js.map
@@ -0,0 +1,5 @@
1
+ import type { HttpRequest, HttpRequestOptions } from '@crawlee/types';
2
+ /**
3
+ * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
4
+ */
5
+ export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
@@ -0,0 +1,36 @@
1
+ import { Readable } from 'node:stream';
2
+ import { applySearchParams } from './url.js';
3
+ /**
4
+ * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
5
+ */
6
+ export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
7
+ const url = new URL(request.url);
8
+ const headers = new Headers(request.headers);
9
+ applySearchParams(url, searchParams);
10
+ if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
11
+ throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
12
+ }
13
+ const body = (() => {
14
+ if (form !== undefined) {
15
+ return Readable.from(new URLSearchParams(form).toString());
16
+ }
17
+ if (json !== undefined) {
18
+ return Readable.from(JSON.stringify(json));
19
+ }
20
+ if (request.body !== undefined) {
21
+ return Readable.from(request.body);
22
+ }
23
+ return undefined;
24
+ })();
25
+ if (form !== undefined && !headers.has('content-type')) {
26
+ headers.set('content-type', 'application/x-www-form-urlencoded');
27
+ }
28
+ if (json !== undefined && !headers.has('content-type')) {
29
+ headers.set('content-type', 'application/json');
30
+ }
31
+ if (username !== undefined || password !== undefined) {
32
+ const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
33
+ headers.set('authorization', `Basic ${encodedAuth}`);
34
+ }
35
+ return { ...request, body, url, headers };
36
+ }
@@ -26,85 +26,6 @@ export declare function isIterable<T>(value: unknown): value is Iterable<T>;
26
26
  * ```
27
27
  */
28
28
  export declare function isAsyncIterable<T>(value: unknown): value is AsyncIterable<T>;
29
- /**
30
- * Converts any iterable or async iterable to an async iterable.
31
- * @internal
32
- *
33
- * @yields Each item from the input iterable
34
- *
35
- * **Example usage:**
36
- * ```ts
37
- * const syncArray = [1, 2, 3];
38
- * for await (const item of asyncifyIterable(syncArray)) {
39
- * console.log(item); // 1, 2, 3
40
- * }
41
- * ```
42
- */
43
- export declare function asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterable<T>;
44
- /**
45
- * Lazily splits the input async iterable into chunks of specified size.
46
- * The last chunk may contain fewer items if the total number of items
47
- * is not evenly divisible by the chunk size.
48
- * @internal
49
- *
50
- * @yields Arrays of items, each containing up to chunkSize items
51
- *
52
- * **Example usage:**
53
- * ```ts
54
- * const numbers = async function* () {
55
- * for (let i = 1; i <= 10; i++) yield i;
56
- * };
57
- *
58
- * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
59
- * console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
60
- * }
61
- * ```
62
- */
63
- export declare function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number): AsyncIterable<T[]>;
64
- /**
65
- * An async iterator that also supports peeking at the next value without consuming it.
66
- * Extends both AsyncIterator and AsyncIterable interfaces.
67
- * @internal
68
- */
69
- export interface PeekableAsyncIterator<T> extends AsyncIterator<T>, AsyncIterable<T> {
70
- /**
71
- * Peeks at the next value without consuming it from the iterator.
72
- * Subsequent calls to peek() will return the same value until next() is called.
73
- *
74
- * @returns Promise that resolves to the next value, or undefined if the iterator is exhausted
75
- */
76
- peek(): Promise<T | undefined>;
77
- }
78
- /**
79
- * An async iterable that yields peekable async iterators.
80
- * @internal
81
- */
82
- export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
83
- [Symbol.asyncIterator](): PeekableAsyncIterator<T>;
84
- }
85
- /**
86
- * Wraps an async iterable to provide peek functionality, allowing you to look at
87
- * the next value without consuming it from the iterator.
88
- * @internal
89
- *
90
- * @param iterable - The async iterable to make peekable
91
- *
92
- * **Example usage:**
93
- * ```ts
94
- * const numbers = async function* () {
95
- * yield 1; yield 2; yield 3;
96
- * };
97
- *
98
- * const peekable = peekableAsyncIterable(numbers());
99
- * const iterator = peekable[Symbol.asyncIterator]();
100
- *
101
- * console.log(await iterator.peek()); // 1 (doesn't consume)
102
- * console.log(await iterator.peek()); // 1 (still doesn't consume)
103
- * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
104
- * console.log(await iterator.peek()); // 2 (next value)
105
- * ```
106
- */
107
- export declare function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T>;
108
29
  /**
109
30
  * Merges multiple async iterables into a single async iterable, yielding values concurrently.
110
31
  *
@@ -124,4 +45,3 @@ export declare function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | It
124
45
  * ```
125
46
  */
126
47
  export declare function mergeAsyncIterables<T>(...iterables: AsyncIterable<T>[]): AsyncIterable<T>;
127
- //# sourceMappingURL=iterables.d.ts.map
@@ -1,4 +1,3 @@
1
- import { inspect } from 'node:util';
2
1
  /**
3
2
  * Type guard that checks if a value is iterable (has Symbol.iterator).
4
3
  * @internal
@@ -40,130 +39,6 @@ export function isAsyncIterable(value) {
40
39
  }
41
40
  return typeof Object(value)[Symbol.asyncIterator] === 'function';
42
41
  }
43
- /**
44
- * Converts any iterable or async iterable to an async iterable.
45
- * @internal
46
- *
47
- * @yields Each item from the input iterable
48
- *
49
- * **Example usage:**
50
- * ```ts
51
- * const syncArray = [1, 2, 3];
52
- * for await (const item of asyncifyIterable(syncArray)) {
53
- * console.log(item); // 1, 2, 3
54
- * }
55
- * ```
56
- */
57
- export async function* asyncifyIterable(iterable) {
58
- yield* iterable;
59
- }
60
- /**
61
- * Lazily splits the input async iterable into chunks of specified size.
62
- * The last chunk may contain fewer items if the total number of items
63
- * is not evenly divisible by the chunk size.
64
- * @internal
65
- *
66
- * @yields Arrays of items, each containing up to chunkSize items
67
- *
68
- * **Example usage:**
69
- * ```ts
70
- * const numbers = async function* () {
71
- * for (let i = 1; i <= 10; i++) yield i;
72
- * };
73
- *
74
- * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
75
- * console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
76
- * }
77
- * ```
78
- */
79
- export async function* chunkedAsyncIterable(iterable, chunkSize) {
80
- if (typeof chunkSize !== 'number' || chunkSize < 1) {
81
- throw new Error(`Chunk size must be a positive number (${inspect(chunkSize)}) received`);
82
- }
83
- let chunk = [];
84
- for await (const item of iterable) {
85
- chunk.push(item);
86
- if (chunk.length >= chunkSize) {
87
- yield chunk;
88
- chunk = [];
89
- }
90
- }
91
- if (chunk.length) {
92
- yield chunk;
93
- }
94
- }
95
- /**
96
- * Wraps an async iterable to provide peek functionality, allowing you to look at
97
- * the next value without consuming it from the iterator.
98
- * @internal
99
- *
100
- * @param iterable - The async iterable to make peekable
101
- *
102
- * **Example usage:**
103
- * ```ts
104
- * const numbers = async function* () {
105
- * yield 1; yield 2; yield 3;
106
- * };
107
- *
108
- * const peekable = peekableAsyncIterable(numbers());
109
- * const iterator = peekable[Symbol.asyncIterator]();
110
- *
111
- * console.log(await iterator.peek()); // 1 (doesn't consume)
112
- * console.log(await iterator.peek()); // 1 (still doesn't consume)
113
- * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
114
- * console.log(await iterator.peek()); // 2 (next value)
115
- * ```
116
- */
117
- export function peekableAsyncIterable(iterable) {
118
- const iterator = asyncifyIterable(iterable)[Symbol.asyncIterator]();
119
- let peekedValue;
120
- let isExhausted = false;
121
- const peekableIterator = {
122
- async next() {
123
- // If we have peeked a value, return it and clear the peek
124
- if (peekedValue !== undefined) {
125
- const result = peekedValue;
126
- peekedValue = undefined;
127
- if (result.done) {
128
- isExhausted = true;
129
- return { done: true, value: undefined };
130
- }
131
- return { done: false, value: result.value };
132
- }
133
- if (isExhausted) {
134
- return { done: true, value: undefined };
135
- }
136
- const result = await iterator.next();
137
- if (result.done) {
138
- isExhausted = true;
139
- }
140
- return result;
141
- },
142
- async peek() {
143
- if (peekedValue !== undefined) {
144
- return peekedValue.done ? undefined : peekedValue.value;
145
- }
146
- if (isExhausted) {
147
- return undefined;
148
- }
149
- const result = await iterator.next();
150
- peekedValue = { done: result.done ?? false, value: result.value };
151
- if (result.done) {
152
- isExhausted = true;
153
- return undefined;
154
- }
155
- return result.value;
156
- },
157
- [Symbol.asyncIterator]() {
158
- return this;
159
- },
160
- };
161
- return {
162
- [Symbol.asyncIterator]() {
163
- return peekableIterator;
164
- },
165
- };
166
- }
167
42
  // Source - https://stackoverflow.com/a/71288323
168
43
  /**
169
44
  * Merges multiple async iterables into a single async iterable, yielding values concurrently.
@@ -219,4 +94,3 @@ export async function* mergeAsyncIterables(...iterables) {
219
94
  }
220
95
  return results;
221
96
  }
222
- //# sourceMappingURL=iterables.js.map
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Records property writes made through the proxies it hands out, applying them to the real objects as
3
+ * they happen, so that a losing attempt's writes can be undone afterwards - on the objects themselves
4
+ * ({@link MutationJournal.revert|`revert()`}), or on a copy of them
5
+ * ({@link MutationJournal.undoOnto|`undoOnto()`}) to reconstruct how they looked before the attempt.
6
+ *
7
+ * An undo on the live object only touches a property that still holds the value the attempt wrote - a
8
+ * value written by someone else in the meantime is left alone and reported via `onConflict`. Writes that
9
+ * bypass the proxy are invisible until they trip that guard.
10
+ */
11
+ export declare class MutationJournal {
12
+ #private;
13
+ track<T extends object>(target: T, path?: PropertyKey[]): T;
14
+ /** Stops recording. Proxies keep writing through. */
15
+ close(): void;
16
+ /** Undoes the recorded writes, newest first, and stops recording. */
17
+ revert({ onConflict }?: {
18
+ onConflict?: (key: PropertyKey) => void;
19
+ }): void;
20
+ /**
21
+ * Applies the inverse of the recorded writes onto `root`, a JSON copy of the tracked object, and returns
22
+ * it. The journal is left as it is. Restored values are JSON-cloned too, so the copy shares nothing with
23
+ * the original and non-JSON values drop out the same way on both.
24
+ *
25
+ * Entries are located by the path under which their object was *first* reached, so an object reachable
26
+ * through two paths is undone under the first one only.
27
+ */
28
+ undoOnto<T extends object>(root: T): T;
29
+ }
@@ -0,0 +1,173 @@
1
+ /**
2
+ * Only plain objects and arrays are wrapped when reached through a tracked object. Built-ins such as `Map`
3
+ * or `Set` reject a proxy receiver in their methods, and the tracked objects are JSON-shaped by contract
4
+ * anyway - such values are handed out as they are, and their mutations are not recorded.
5
+ */
6
+ const isTrackable = (value) => Array.isArray(value) ||
7
+ (typeof value === 'object' &&
8
+ value !== null &&
9
+ (Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null));
10
+ /** Every proxy any journal has handed out, keyed by the proxy. Shared, as a value read through one attempt's
11
+ * proxy may be written back through another's. */
12
+ const proxyTargets = new WeakMap();
13
+ /**
14
+ * Replaces journal proxies in `value` with the objects behind them, in place for nested ones. A value read
15
+ * through a proxy and written back (`state.x = state.x ?? {}`) would otherwise store the proxy itself, and
16
+ * every later attempt would wrap it once more.
17
+ */
18
+ function unwrap(value, seen = new Set()) {
19
+ const target = typeof value === 'object' && value !== null ? proxyTargets.get(value) : undefined;
20
+ if (target)
21
+ return target;
22
+ if (!isTrackable(value) || seen.has(value))
23
+ return value;
24
+ seen.add(value);
25
+ for (const key of Object.keys(value)) {
26
+ const inner = value[key];
27
+ const unwrapped = unwrap(inner, seen);
28
+ if (unwrapped !== inner)
29
+ value[key] = unwrapped;
30
+ }
31
+ return value;
32
+ }
33
+ /**
34
+ * Records property writes made through the proxies it hands out, applying them to the real objects as
35
+ * they happen, so that a losing attempt's writes can be undone afterwards - on the objects themselves
36
+ * ({@link MutationJournal.revert|`revert()`}), or on a copy of them
37
+ * ({@link MutationJournal.undoOnto|`undoOnto()`}) to reconstruct how they looked before the attempt.
38
+ *
39
+ * An undo on the live object only touches a property that still holds the value the attempt wrote - a
40
+ * value written by someone else in the meantime is left alone and reported via `onConflict`. Writes that
41
+ * bypass the proxy are invisible until they trip that guard.
42
+ */
43
+ export class MutationJournal {
44
+ #entries = [];
45
+ #proxies = new WeakMap();
46
+ #paths = new WeakMap();
47
+ #closed = false;
48
+ track(target, path = []) {
49
+ const cached = this.#proxies.get(target);
50
+ if (cached)
51
+ return cached;
52
+ this.#paths.set(target, path);
53
+ const entry = (key, fields) => this.#entries.push({ target, path, key, ...fields });
54
+ // Accessors run with `this` as the real object, not the proxy: class accessors may touch `#private`
55
+ // members, which throw on a proxy receiver. An accessor write is therefore journaled as its own
56
+ // entry and undone through the same setter, rather than through whatever it writes underneath.
57
+ const proxy = new Proxy(target, {
58
+ get: (object, key) => {
59
+ const value = Reflect.get(object, key);
60
+ return isTrackable(value) ? this.track(value, [...path, key]) : value;
61
+ },
62
+ set: (object, key, rawNext) => {
63
+ const next = unwrap(rawNext);
64
+ const had = Reflect.has(object, key);
65
+ const prev = Reflect.get(object, key);
66
+ this.#recordArrayTruncation(object, key, next);
67
+ if (!Reflect.set(object, key, next))
68
+ return false;
69
+ // An array's `length` is implied by its elements and never undone on its own.
70
+ if (!this.#closed && !(Array.isArray(object) && key === 'length')) {
71
+ entry(key, { had, prev, next, deleted: false });
72
+ }
73
+ return true;
74
+ },
75
+ deleteProperty: (object, key) => {
76
+ const had = Reflect.has(object, key);
77
+ const prev = Reflect.get(object, key);
78
+ if (!Reflect.deleteProperty(object, key))
79
+ return false;
80
+ if (!this.#closed && had)
81
+ entry(key, { had, prev, next: undefined, deleted: true });
82
+ return true;
83
+ },
84
+ });
85
+ this.#proxies.set(target, proxy);
86
+ proxyTargets.set(proxy, target);
87
+ return proxy;
88
+ }
89
+ /**
90
+ * Shrinking an array's `length` drops elements without a trap firing for them. They are recorded as
91
+ * deletions, from the highest index down, so the reverse replay reinserts them from the lowest up.
92
+ */
93
+ #recordArrayTruncation(object, key, next) {
94
+ if (this.#closed || !Array.isArray(object) || key !== 'length')
95
+ return;
96
+ const path = this.#paths.get(object);
97
+ for (let i = object.length - 1; i >= next; i--) {
98
+ if (i in object) {
99
+ this.#entries.push({
100
+ target: object,
101
+ path,
102
+ key: String(i),
103
+ had: true,
104
+ prev: object[i],
105
+ next: undefined,
106
+ deleted: true,
107
+ });
108
+ }
109
+ }
110
+ }
111
+ /**
112
+ * Restores one entry on `target`. Array elements are handled positionally - an added element is removed
113
+ * and a removed one reinserted, shifting the rest - so the result stays something a sequence of `push`
114
+ * and `splice` calls could have produced, even next to elements another writer added meanwhile.
115
+ */
116
+ static #undo(target, { key, had, deleted }, value) {
117
+ if (Array.isArray(target) && typeof key === 'string' && Number.isInteger(Number(key))) {
118
+ if (deleted) {
119
+ target.splice(Number(key), 0, value);
120
+ }
121
+ else if (had) {
122
+ target[Number(key)] = value;
123
+ }
124
+ else {
125
+ target.splice(Number(key), 1);
126
+ }
127
+ return;
128
+ }
129
+ if (had) {
130
+ Reflect.set(target, key, value);
131
+ }
132
+ else {
133
+ Reflect.deleteProperty(target, key);
134
+ }
135
+ }
136
+ /** Stops recording. Proxies keep writing through. */
137
+ close() {
138
+ this.#closed = true;
139
+ }
140
+ /** Undoes the recorded writes, newest first, and stops recording. */
141
+ revert({ onConflict } = {}) {
142
+ this.close();
143
+ for (let i = this.#entries.length - 1; i >= 0; i--) {
144
+ const entry = this.#entries[i];
145
+ const { target, key, next, deleted } = entry;
146
+ const untouched = deleted ? !Reflect.has(target, key) : Reflect.get(target, key) === next;
147
+ if (!untouched) {
148
+ onConflict?.(key);
149
+ continue;
150
+ }
151
+ MutationJournal.#undo(target, entry, entry.prev);
152
+ }
153
+ this.#entries.length = 0;
154
+ }
155
+ /**
156
+ * Applies the inverse of the recorded writes onto `root`, a JSON copy of the tracked object, and returns
157
+ * it. The journal is left as it is. Restored values are JSON-cloned too, so the copy shares nothing with
158
+ * the original and non-JSON values drop out the same way on both.
159
+ *
160
+ * Entries are located by the path under which their object was *first* reached, so an object reachable
161
+ * through two paths is undone under the first one only.
162
+ */
163
+ undoOnto(root) {
164
+ for (let i = this.#entries.length - 1; i >= 0; i--) {
165
+ const entry = this.#entries[i];
166
+ const target = entry.path.reduce((object, step) => object?.[step], root);
167
+ if (typeof target !== 'object' || target === null)
168
+ continue;
169
+ MutationJournal.#undo(target, entry, isTrackable(entry.prev) ? JSON.parse(JSON.stringify(entry.prev)) : entry.prev);
170
+ }
171
+ return root;
172
+ }
173
+ }
@@ -14,7 +14,6 @@ type OpenGraphResult = string | string[] | Dictionary<string | Dictionary>;
14
14
  * Currently existing properties are kept up to date.
15
15
  * @returns Scraped OpenGraph properties as an object.
16
16
  */
17
- export declare function parseOpenGraph(raw: string, additionalProperties?: OpenGraphProperty[]): Dictionary<OpenGraphResult>;
18
- export declare function parseOpenGraph($: CheerioAPI, additionalProperties?: OpenGraphProperty[]): Dictionary<OpenGraphResult>;
17
+ export declare function parseOpenGraph(raw: string, additionalProperties?: OpenGraphProperty[]): Promise<Dictionary<OpenGraphResult>>;
18
+ export declare function parseOpenGraph($: CheerioAPI, additionalProperties?: OpenGraphProperty[]): Promise<Dictionary<OpenGraphResult>>;
19
19
  export {};
20
- //# sourceMappingURL=open_graph_parser.d.ts.map
@@ -1,4 +1,3 @@
1
- import { load } from 'cheerio';
2
1
  /**
3
2
  * To be used with the spread operator. Ensures that the item is defined, and is not empty.
4
3
  *
@@ -255,32 +254,32 @@ const OPEN_GRAPH_PROPERTIES = [
255
254
  outputName: 'articleInfo',
256
255
  children: [
257
256
  {
258
- name: 'music:published_time',
257
+ name: 'article:published_time',
259
258
  outputName: 'publishedTime',
260
259
  children: [],
261
260
  },
262
261
  {
263
- name: 'music:modified_time',
262
+ name: 'article:modified_time',
264
263
  outputName: 'modifiedTime',
265
264
  children: [],
266
265
  },
267
266
  {
268
- name: 'music:expiration_time',
267
+ name: 'article:expiration_time',
269
268
  outputName: 'expirationTime',
270
269
  children: [],
271
270
  },
272
271
  {
273
- name: 'music:author',
272
+ name: 'article:author',
274
273
  outputName: 'author',
275
274
  children: [],
276
275
  },
277
276
  {
278
- name: 'music:section',
277
+ name: 'article:section',
279
278
  outputName: 'section',
280
279
  children: [],
281
280
  },
282
281
  {
283
- name: 'music:tag',
282
+ name: 'article:tag',
284
283
  outputName: 'tag',
285
284
  children: [],
286
285
  },
@@ -366,7 +365,8 @@ const parseOpenGraphProperty = (property, $) => {
366
365
  }, {}),
367
366
  };
368
367
  };
369
- export function parseOpenGraph(item, additionalProperties) {
368
+ export async function parseOpenGraph(item, additionalProperties) {
369
+ const { load } = await import('cheerio/slim');
370
370
  const $ = typeof item === 'string' ? load(item) : item;
371
371
  return [...(additionalProperties || []), ...OPEN_GRAPH_PROPERTIES].reduce((acc, curr) => {
372
372
  return {
@@ -375,4 +375,3 @@ export function parseOpenGraph(item, additionalProperties) {
375
375
  };
376
376
  }, {});
377
377
  }
378
- //# sourceMappingURL=open_graph_parser.js.map