@crawlee/utils 4.0.0-beta.25 → 4.0.0-beta.251
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +10 -17
- package/index.js +8 -16
- package/internal.d.ts +11 -0
- package/internal.js +10 -0
- package/internals/blocked.d.ts +0 -1
- package/internals/blocked.js +0 -1
- package/internals/cheerio.d.ts +1 -3
- package/internals/cheerio.js +25 -16
- package/internals/extract-microdata.d.ts +24 -0
- package/internals/extract-microdata.js +118 -0
- package/internals/extract-urls.d.ts +1 -2
- package/internals/extract-urls.js +18 -17
- package/internals/general.d.ts +0 -25
- package/internals/general.js +2 -110
- package/internals/iterables.d.ts +0 -80
- package/internals/iterables.js +0 -126
- package/internals/mutation-journal.d.ts +29 -0
- package/internals/mutation-journal.js +173 -0
- package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
- package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
- package/internals/robots.d.ts +35 -15
- package/internals/robots.js +66 -30
- package/internals/schemas.d.ts +114 -0
- package/internals/schemas.js +115 -0
- package/internals/sitemap.d.ts +44 -3
- package/internals/sitemap.js +171 -85
- package/internals/social.d.ts +2 -3
- package/internals/social.js +8 -6
- package/internals/url.d.ts +69 -1
- package/internals/url.js +120 -2
- package/internals/validation.d.ts +9 -0
- package/internals/validation.js +16 -0
- package/package.json +16 -15
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/blocked.d.ts.map +0 -1
- package/internals/blocked.js.map +0 -1
- package/internals/cheerio.d.ts.map +0 -1
- package/internals/cheerio.js.map +0 -1
- package/internals/chunk.d.ts +0 -2
- package/internals/chunk.d.ts.map +0 -1
- package/internals/chunk.js +0 -40
- package/internals/chunk.js.map +0 -1
- package/internals/debug.d.ts +0 -32
- package/internals/debug.d.ts.map +0 -1
- package/internals/debug.js +0 -41
- package/internals/debug.js.map +0 -1
- package/internals/extract-urls.d.ts.map +0 -1
- package/internals/extract-urls.js.map +0 -1
- package/internals/general.d.ts.map +0 -1
- package/internals/general.js.map +0 -1
- package/internals/iterables.d.ts.map +0 -1
- package/internals/iterables.js.map +0 -1
- package/internals/open_graph_parser.d.ts.map +0 -1
- package/internals/open_graph_parser.js.map +0 -1
- package/internals/robots.d.ts.map +0 -1
- package/internals/robots.js.map +0 -1
- package/internals/sitemap.d.ts.map +0 -1
- package/internals/sitemap.js.map +0 -1
- package/internals/social.d.ts.map +0 -1
- package/internals/social.js.map +0 -1
- package/internals/system-info/cpu-info.d.ts +0 -64
- package/internals/system-info/cpu-info.d.ts.map +0 -1
- package/internals/system-info/cpu-info.js +0 -211
- package/internals/system-info/cpu-info.js.map +0 -1
- package/internals/system-info/memory-info.d.ts +0 -28
- package/internals/system-info/memory-info.d.ts.map +0 -1
- package/internals/system-info/memory-info.js +0 -118
- package/internals/system-info/memory-info.js.map +0 -1
- package/internals/system-info/ps-tree.d.ts +0 -18
- package/internals/system-info/ps-tree.d.ts.map +0 -1
- package/internals/system-info/ps-tree.js +0 -145
- package/internals/system-info/ps-tree.js.map +0 -1
- package/internals/typedefs.d.ts +0 -5
- package/internals/typedefs.d.ts.map +0 -1
- package/internals/typedefs.js +0 -9
- package/internals/typedefs.js.map +0 -1
- package/internals/url.d.ts.map +0 -1
- package/internals/url.js.map +0 -1
package/internals/general.js
CHANGED
|
@@ -1,108 +1,14 @@
|
|
|
1
|
-
import fs from 'node:fs/promises';
|
|
2
1
|
import { setTimeout } from 'node:timers/promises';
|
|
3
2
|
/**
|
|
4
3
|
* Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
|
|
5
4
|
* and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
|
|
6
5
|
*/
|
|
7
|
-
export const URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()]*)?/giu;
|
|
6
|
+
export const URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()'*]*)?/giu;
|
|
8
7
|
/**
|
|
9
8
|
* Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
|
|
10
9
|
* Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
|
|
11
10
|
*/
|
|
12
|
-
export const URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()]*)?/giu;
|
|
13
|
-
let isDockerPromiseCache;
|
|
14
|
-
async function createIsDockerPromise() {
|
|
15
|
-
const promise1 = fs
|
|
16
|
-
.stat('/.dockerenv')
|
|
17
|
-
.then(() => true)
|
|
18
|
-
.catch(() => false);
|
|
19
|
-
const promise2 = fs
|
|
20
|
-
.readFile('/proc/self/cgroup', 'utf8')
|
|
21
|
-
.then((content) => content.includes('docker'))
|
|
22
|
-
.catch(() => false);
|
|
23
|
-
const [result1, result2] = await Promise.all([promise1, promise2]);
|
|
24
|
-
return result1 || result2;
|
|
25
|
-
}
|
|
26
|
-
/**
|
|
27
|
-
* Returns a `Promise` that resolves to true if the code is running in a Docker container.
|
|
28
|
-
*/
|
|
29
|
-
export async function isDocker(forceReset) {
|
|
30
|
-
// Parameter forceReset is just internal for unit tests.
|
|
31
|
-
if (!isDockerPromiseCache || forceReset)
|
|
32
|
-
isDockerPromiseCache = createIsDockerPromise();
|
|
33
|
-
return isDockerPromiseCache;
|
|
34
|
-
}
|
|
35
|
-
let isContainerizedResult;
|
|
36
|
-
/**
|
|
37
|
-
* Detects if crawlee is running in a containerized environment.
|
|
38
|
-
*/
|
|
39
|
-
export async function isContainerized() {
|
|
40
|
-
// Value is very unlikley to change. Cache the result after the first execution.
|
|
41
|
-
if (isContainerizedResult !== undefined) {
|
|
42
|
-
return isContainerizedResult;
|
|
43
|
-
}
|
|
44
|
-
// return false if running in aws lambda
|
|
45
|
-
if (isLambda()) {
|
|
46
|
-
isContainerizedResult = false;
|
|
47
|
-
return isContainerizedResult;
|
|
48
|
-
}
|
|
49
|
-
const dockerenvCheck = fs
|
|
50
|
-
.stat('/.dockerenv')
|
|
51
|
-
.then(() => true)
|
|
52
|
-
.catch(() => false);
|
|
53
|
-
const cgroupCheck = fs
|
|
54
|
-
.readFile('/proc/self/cgroup', 'utf8')
|
|
55
|
-
.then((content) => content.includes('docker'))
|
|
56
|
-
.catch(() => false);
|
|
57
|
-
const [dockerenvResult, cgroupResult] = await Promise.all([dockerenvCheck, cgroupCheck]);
|
|
58
|
-
isContainerizedResult = dockerenvResult || cgroupResult || !!process.env.KUBERNETES_SERVICE_HOST;
|
|
59
|
-
return isContainerizedResult;
|
|
60
|
-
}
|
|
61
|
-
export function isLambda() {
|
|
62
|
-
return !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
|
|
63
|
-
}
|
|
64
|
-
let _cgroupsVersion;
|
|
65
|
-
/**
|
|
66
|
-
* gets the cgroup version by checking for a file at /sys/fs/cgroup/memory
|
|
67
|
-
* @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.
|
|
68
|
-
*/
|
|
69
|
-
export async function getCgroupsVersion(forceReset) {
|
|
70
|
-
// Parameter forceReset is just internal for unit tests.
|
|
71
|
-
if (_cgroupsVersion !== undefined && !forceReset) {
|
|
72
|
-
return _cgroupsVersion;
|
|
73
|
-
}
|
|
74
|
-
try {
|
|
75
|
-
// If this directory does not exists, cgroups are not available
|
|
76
|
-
await fs.access('/sys/fs/cgroup/');
|
|
77
|
-
}
|
|
78
|
-
catch (e) {
|
|
79
|
-
_cgroupsVersion = null;
|
|
80
|
-
return null;
|
|
81
|
-
}
|
|
82
|
-
_cgroupsVersion = 'V1';
|
|
83
|
-
try {
|
|
84
|
-
// If this directory does not exists, assume the container is using cgroups V2
|
|
85
|
-
await fs.access('/sys/fs/cgroup/memory/');
|
|
86
|
-
}
|
|
87
|
-
catch (e) {
|
|
88
|
-
_cgroupsVersion = 'V2';
|
|
89
|
-
}
|
|
90
|
-
return _cgroupsVersion;
|
|
91
|
-
}
|
|
92
|
-
/**
|
|
93
|
-
* Computes a weighted average of an array of numbers, complemented by an array of weights.
|
|
94
|
-
* @ignore
|
|
95
|
-
*/
|
|
96
|
-
export function weightedAvg(arrValues, arrWeights) {
|
|
97
|
-
const result = arrValues
|
|
98
|
-
.map((value, i) => {
|
|
99
|
-
const weight = arrWeights[i];
|
|
100
|
-
const sum = value * weight;
|
|
101
|
-
return [sum, weight];
|
|
102
|
-
})
|
|
103
|
-
.reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);
|
|
104
|
-
return result[0] / result[1];
|
|
105
|
-
}
|
|
11
|
+
export const URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;
|
|
106
12
|
/**
|
|
107
13
|
* Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
|
|
108
14
|
* in your code, e.g. to prevent overloading of target website or to avoid bot detection.
|
|
@@ -122,19 +28,6 @@ export function weightedAvg(arrValues, arrWeights) {
|
|
|
122
28
|
export async function sleep(millis) {
|
|
123
29
|
return setTimeout(millis ?? undefined);
|
|
124
30
|
}
|
|
125
|
-
/**
|
|
126
|
-
* Converts SNAKE_CASE to camelCase.
|
|
127
|
-
* @ignore
|
|
128
|
-
*/
|
|
129
|
-
export function snakeCaseToCamelCase(snakeCaseStr) {
|
|
130
|
-
return snakeCaseStr
|
|
131
|
-
.toLowerCase()
|
|
132
|
-
.split('_')
|
|
133
|
-
.map((part, index) => {
|
|
134
|
-
return index > 0 ? part.charAt(0).toUpperCase() + part.slice(1) : part;
|
|
135
|
-
})
|
|
136
|
-
.join('');
|
|
137
|
-
}
|
|
138
31
|
/**
|
|
139
32
|
* Traverses DOM and expands shadow-root elements (created by custom components).
|
|
140
33
|
* @ignore
|
|
@@ -164,4 +57,3 @@ export function expandShadowRoots(document) {
|
|
|
164
57
|
replaceShadowDomsWithHtml(document.body);
|
|
165
58
|
return document.documentElement.outerHTML;
|
|
166
59
|
}
|
|
167
|
-
//# sourceMappingURL=general.js.map
|
package/internals/iterables.d.ts
CHANGED
|
@@ -26,85 +26,6 @@ export declare function isIterable<T>(value: unknown): value is Iterable<T>;
|
|
|
26
26
|
* ```
|
|
27
27
|
*/
|
|
28
28
|
export declare function isAsyncIterable<T>(value: unknown): value is AsyncIterable<T>;
|
|
29
|
-
/**
|
|
30
|
-
* Converts any iterable or async iterable to an async iterable.
|
|
31
|
-
* @internal
|
|
32
|
-
*
|
|
33
|
-
* @yields Each item from the input iterable
|
|
34
|
-
*
|
|
35
|
-
* **Example usage:**
|
|
36
|
-
* ```ts
|
|
37
|
-
* const syncArray = [1, 2, 3];
|
|
38
|
-
* for await (const item of asyncifyIterable(syncArray)) {
|
|
39
|
-
* console.log(item); // 1, 2, 3
|
|
40
|
-
* }
|
|
41
|
-
* ```
|
|
42
|
-
*/
|
|
43
|
-
export declare function asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterable<T>;
|
|
44
|
-
/**
|
|
45
|
-
* Lazily splits the input async iterable into chunks of specified size.
|
|
46
|
-
* The last chunk may contain fewer items if the total number of items
|
|
47
|
-
* is not evenly divisible by the chunk size.
|
|
48
|
-
* @internal
|
|
49
|
-
*
|
|
50
|
-
* @yields Arrays of items, each containing up to chunkSize items
|
|
51
|
-
*
|
|
52
|
-
* **Example usage:**
|
|
53
|
-
* ```ts
|
|
54
|
-
* const numbers = async function* () {
|
|
55
|
-
* for (let i = 1; i <= 10; i++) yield i;
|
|
56
|
-
* };
|
|
57
|
-
*
|
|
58
|
-
* for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
|
|
59
|
-
* console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
|
|
60
|
-
* }
|
|
61
|
-
* ```
|
|
62
|
-
*/
|
|
63
|
-
export declare function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number): AsyncIterable<T[]>;
|
|
64
|
-
/**
|
|
65
|
-
* An async iterator that also supports peeking at the next value without consuming it.
|
|
66
|
-
* Extends both AsyncIterator and AsyncIterable interfaces.
|
|
67
|
-
* @internal
|
|
68
|
-
*/
|
|
69
|
-
export interface PeekableAsyncIterator<T> extends AsyncIterator<T>, AsyncIterable<T> {
|
|
70
|
-
/**
|
|
71
|
-
* Peeks at the next value without consuming it from the iterator.
|
|
72
|
-
* Subsequent calls to peek() will return the same value until next() is called.
|
|
73
|
-
*
|
|
74
|
-
* @returns Promise that resolves to the next value, or undefined if the iterator is exhausted
|
|
75
|
-
*/
|
|
76
|
-
peek(): Promise<T | undefined>;
|
|
77
|
-
}
|
|
78
|
-
/**
|
|
79
|
-
* An async iterable that yields peekable async iterators.
|
|
80
|
-
* @internal
|
|
81
|
-
*/
|
|
82
|
-
export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
|
|
83
|
-
[Symbol.asyncIterator](): PeekableAsyncIterator<T>;
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* Wraps an async iterable to provide peek functionality, allowing you to look at
|
|
87
|
-
* the next value without consuming it from the iterator.
|
|
88
|
-
* @internal
|
|
89
|
-
*
|
|
90
|
-
* @param iterable - The async iterable to make peekable
|
|
91
|
-
*
|
|
92
|
-
* **Example usage:**
|
|
93
|
-
* ```ts
|
|
94
|
-
* const numbers = async function* () {
|
|
95
|
-
* yield 1; yield 2; yield 3;
|
|
96
|
-
* };
|
|
97
|
-
*
|
|
98
|
-
* const peekable = peekableAsyncIterable(numbers());
|
|
99
|
-
* const iterator = peekable[Symbol.asyncIterator]();
|
|
100
|
-
*
|
|
101
|
-
* console.log(await iterator.peek()); // 1 (doesn't consume)
|
|
102
|
-
* console.log(await iterator.peek()); // 1 (still doesn't consume)
|
|
103
|
-
* console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
|
|
104
|
-
* console.log(await iterator.peek()); // 2 (next value)
|
|
105
|
-
* ```
|
|
106
|
-
*/
|
|
107
|
-
export declare function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T>;
|
|
108
29
|
/**
|
|
109
30
|
* Merges multiple async iterables into a single async iterable, yielding values concurrently.
|
|
110
31
|
*
|
|
@@ -124,4 +45,3 @@ export declare function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | It
|
|
|
124
45
|
* ```
|
|
125
46
|
*/
|
|
126
47
|
export declare function mergeAsyncIterables<T>(...iterables: AsyncIterable<T>[]): AsyncIterable<T>;
|
|
127
|
-
//# sourceMappingURL=iterables.d.ts.map
|
package/internals/iterables.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { inspect } from 'node:util';
|
|
2
1
|
/**
|
|
3
2
|
* Type guard that checks if a value is iterable (has Symbol.iterator).
|
|
4
3
|
* @internal
|
|
@@ -40,130 +39,6 @@ export function isAsyncIterable(value) {
|
|
|
40
39
|
}
|
|
41
40
|
return typeof Object(value)[Symbol.asyncIterator] === 'function';
|
|
42
41
|
}
|
|
43
|
-
/**
|
|
44
|
-
* Converts any iterable or async iterable to an async iterable.
|
|
45
|
-
* @internal
|
|
46
|
-
*
|
|
47
|
-
* @yields Each item from the input iterable
|
|
48
|
-
*
|
|
49
|
-
* **Example usage:**
|
|
50
|
-
* ```ts
|
|
51
|
-
* const syncArray = [1, 2, 3];
|
|
52
|
-
* for await (const item of asyncifyIterable(syncArray)) {
|
|
53
|
-
* console.log(item); // 1, 2, 3
|
|
54
|
-
* }
|
|
55
|
-
* ```
|
|
56
|
-
*/
|
|
57
|
-
export async function* asyncifyIterable(iterable) {
|
|
58
|
-
yield* iterable;
|
|
59
|
-
}
|
|
60
|
-
/**
|
|
61
|
-
* Lazily splits the input async iterable into chunks of specified size.
|
|
62
|
-
* The last chunk may contain fewer items if the total number of items
|
|
63
|
-
* is not evenly divisible by the chunk size.
|
|
64
|
-
* @internal
|
|
65
|
-
*
|
|
66
|
-
* @yields Arrays of items, each containing up to chunkSize items
|
|
67
|
-
*
|
|
68
|
-
* **Example usage:**
|
|
69
|
-
* ```ts
|
|
70
|
-
* const numbers = async function* () {
|
|
71
|
-
* for (let i = 1; i <= 10; i++) yield i;
|
|
72
|
-
* };
|
|
73
|
-
*
|
|
74
|
-
* for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
|
|
75
|
-
* console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
|
|
76
|
-
* }
|
|
77
|
-
* ```
|
|
78
|
-
*/
|
|
79
|
-
export async function* chunkedAsyncIterable(iterable, chunkSize) {
|
|
80
|
-
if (typeof chunkSize !== 'number' || chunkSize < 1) {
|
|
81
|
-
throw new Error(`Chunk size must be a positive number (${inspect(chunkSize)}) received`);
|
|
82
|
-
}
|
|
83
|
-
let chunk = [];
|
|
84
|
-
for await (const item of iterable) {
|
|
85
|
-
chunk.push(item);
|
|
86
|
-
if (chunk.length >= chunkSize) {
|
|
87
|
-
yield chunk;
|
|
88
|
-
chunk = [];
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
if (chunk.length) {
|
|
92
|
-
yield chunk;
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
/**
|
|
96
|
-
* Wraps an async iterable to provide peek functionality, allowing you to look at
|
|
97
|
-
* the next value without consuming it from the iterator.
|
|
98
|
-
* @internal
|
|
99
|
-
*
|
|
100
|
-
* @param iterable - The async iterable to make peekable
|
|
101
|
-
*
|
|
102
|
-
* **Example usage:**
|
|
103
|
-
* ```ts
|
|
104
|
-
* const numbers = async function* () {
|
|
105
|
-
* yield 1; yield 2; yield 3;
|
|
106
|
-
* };
|
|
107
|
-
*
|
|
108
|
-
* const peekable = peekableAsyncIterable(numbers());
|
|
109
|
-
* const iterator = peekable[Symbol.asyncIterator]();
|
|
110
|
-
*
|
|
111
|
-
* console.log(await iterator.peek()); // 1 (doesn't consume)
|
|
112
|
-
* console.log(await iterator.peek()); // 1 (still doesn't consume)
|
|
113
|
-
* console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
|
|
114
|
-
* console.log(await iterator.peek()); // 2 (next value)
|
|
115
|
-
* ```
|
|
116
|
-
*/
|
|
117
|
-
export function peekableAsyncIterable(iterable) {
|
|
118
|
-
const iterator = asyncifyIterable(iterable)[Symbol.asyncIterator]();
|
|
119
|
-
let peekedValue;
|
|
120
|
-
let isExhausted = false;
|
|
121
|
-
const peekableIterator = {
|
|
122
|
-
async next() {
|
|
123
|
-
// If we have peeked a value, return it and clear the peek
|
|
124
|
-
if (peekedValue !== undefined) {
|
|
125
|
-
const result = peekedValue;
|
|
126
|
-
peekedValue = undefined;
|
|
127
|
-
if (result.done) {
|
|
128
|
-
isExhausted = true;
|
|
129
|
-
return { done: true, value: undefined };
|
|
130
|
-
}
|
|
131
|
-
return { done: false, value: result.value };
|
|
132
|
-
}
|
|
133
|
-
if (isExhausted) {
|
|
134
|
-
return { done: true, value: undefined };
|
|
135
|
-
}
|
|
136
|
-
const result = await iterator.next();
|
|
137
|
-
if (result.done) {
|
|
138
|
-
isExhausted = true;
|
|
139
|
-
}
|
|
140
|
-
return result;
|
|
141
|
-
},
|
|
142
|
-
async peek() {
|
|
143
|
-
if (peekedValue !== undefined) {
|
|
144
|
-
return peekedValue.done ? undefined : peekedValue.value;
|
|
145
|
-
}
|
|
146
|
-
if (isExhausted) {
|
|
147
|
-
return undefined;
|
|
148
|
-
}
|
|
149
|
-
const result = await iterator.next();
|
|
150
|
-
peekedValue = { done: result.done ?? false, value: result.value };
|
|
151
|
-
if (result.done) {
|
|
152
|
-
isExhausted = true;
|
|
153
|
-
return undefined;
|
|
154
|
-
}
|
|
155
|
-
return result.value;
|
|
156
|
-
},
|
|
157
|
-
[Symbol.asyncIterator]() {
|
|
158
|
-
return this;
|
|
159
|
-
},
|
|
160
|
-
};
|
|
161
|
-
return {
|
|
162
|
-
[Symbol.asyncIterator]() {
|
|
163
|
-
return peekableIterator;
|
|
164
|
-
},
|
|
165
|
-
};
|
|
166
|
-
}
|
|
167
42
|
// Source - https://stackoverflow.com/a/71288323
|
|
168
43
|
/**
|
|
169
44
|
* Merges multiple async iterables into a single async iterable, yielding values concurrently.
|
|
@@ -219,4 +94,3 @@ export async function* mergeAsyncIterables(...iterables) {
|
|
|
219
94
|
}
|
|
220
95
|
return results;
|
|
221
96
|
}
|
|
222
|
-
//# sourceMappingURL=iterables.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Records property writes made through the proxies it hands out, applying them to the real objects as
|
|
3
|
+
* they happen, so that a losing attempt's writes can be undone afterwards - on the objects themselves
|
|
4
|
+
* ({@link MutationJournal.revert|`revert()`}), or on a copy of them
|
|
5
|
+
* ({@link MutationJournal.undoOnto|`undoOnto()`}) to reconstruct how they looked before the attempt.
|
|
6
|
+
*
|
|
7
|
+
* An undo on the live object only touches a property that still holds the value the attempt wrote - a
|
|
8
|
+
* value written by someone else in the meantime is left alone and reported via `onConflict`. Writes that
|
|
9
|
+
* bypass the proxy are invisible until they trip that guard.
|
|
10
|
+
*/
|
|
11
|
+
export declare class MutationJournal {
|
|
12
|
+
#private;
|
|
13
|
+
track<T extends object>(target: T, path?: PropertyKey[]): T;
|
|
14
|
+
/** Stops recording. Proxies keep writing through. */
|
|
15
|
+
close(): void;
|
|
16
|
+
/** Undoes the recorded writes, newest first, and stops recording. */
|
|
17
|
+
revert({ onConflict }?: {
|
|
18
|
+
onConflict?: (key: PropertyKey) => void;
|
|
19
|
+
}): void;
|
|
20
|
+
/**
|
|
21
|
+
* Applies the inverse of the recorded writes onto `root`, a JSON copy of the tracked object, and returns
|
|
22
|
+
* it. The journal is left as it is. Restored values are JSON-cloned too, so the copy shares nothing with
|
|
23
|
+
* the original and non-JSON values drop out the same way on both.
|
|
24
|
+
*
|
|
25
|
+
* Entries are located by the path under which their object was *first* reached, so an object reachable
|
|
26
|
+
* through two paths is undone under the first one only.
|
|
27
|
+
*/
|
|
28
|
+
undoOnto<T extends object>(root: T): T;
|
|
29
|
+
}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Only plain objects and arrays are wrapped when reached through a tracked object. Built-ins such as `Map`
|
|
3
|
+
* or `Set` reject a proxy receiver in their methods, and the tracked objects are JSON-shaped by contract
|
|
4
|
+
* anyway - such values are handed out as they are, and their mutations are not recorded.
|
|
5
|
+
*/
|
|
6
|
+
const isTrackable = (value) => Array.isArray(value) ||
|
|
7
|
+
(typeof value === 'object' &&
|
|
8
|
+
value !== null &&
|
|
9
|
+
(Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null));
|
|
10
|
+
/** Every proxy any journal has handed out, keyed by the proxy. Shared, as a value read through one attempt's
|
|
11
|
+
* proxy may be written back through another's. */
|
|
12
|
+
const proxyTargets = new WeakMap();
|
|
13
|
+
/**
|
|
14
|
+
* Replaces journal proxies in `value` with the objects behind them, in place for nested ones. A value read
|
|
15
|
+
* through a proxy and written back (`state.x = state.x ?? {}`) would otherwise store the proxy itself, and
|
|
16
|
+
* every later attempt would wrap it once more.
|
|
17
|
+
*/
|
|
18
|
+
function unwrap(value, seen = new Set()) {
|
|
19
|
+
const target = typeof value === 'object' && value !== null ? proxyTargets.get(value) : undefined;
|
|
20
|
+
if (target)
|
|
21
|
+
return target;
|
|
22
|
+
if (!isTrackable(value) || seen.has(value))
|
|
23
|
+
return value;
|
|
24
|
+
seen.add(value);
|
|
25
|
+
for (const key of Object.keys(value)) {
|
|
26
|
+
const inner = value[key];
|
|
27
|
+
const unwrapped = unwrap(inner, seen);
|
|
28
|
+
if (unwrapped !== inner)
|
|
29
|
+
value[key] = unwrapped;
|
|
30
|
+
}
|
|
31
|
+
return value;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Records property writes made through the proxies it hands out, applying them to the real objects as
|
|
35
|
+
* they happen, so that a losing attempt's writes can be undone afterwards - on the objects themselves
|
|
36
|
+
* ({@link MutationJournal.revert|`revert()`}), or on a copy of them
|
|
37
|
+
* ({@link MutationJournal.undoOnto|`undoOnto()`}) to reconstruct how they looked before the attempt.
|
|
38
|
+
*
|
|
39
|
+
* An undo on the live object only touches a property that still holds the value the attempt wrote - a
|
|
40
|
+
* value written by someone else in the meantime is left alone and reported via `onConflict`. Writes that
|
|
41
|
+
* bypass the proxy are invisible until they trip that guard.
|
|
42
|
+
*/
|
|
43
|
+
export class MutationJournal {
|
|
44
|
+
#entries = [];
|
|
45
|
+
#proxies = new WeakMap();
|
|
46
|
+
#paths = new WeakMap();
|
|
47
|
+
#closed = false;
|
|
48
|
+
track(target, path = []) {
|
|
49
|
+
const cached = this.#proxies.get(target);
|
|
50
|
+
if (cached)
|
|
51
|
+
return cached;
|
|
52
|
+
this.#paths.set(target, path);
|
|
53
|
+
const entry = (key, fields) => this.#entries.push({ target, path, key, ...fields });
|
|
54
|
+
// Accessors run with `this` as the real object, not the proxy: class accessors may touch `#private`
|
|
55
|
+
// members, which throw on a proxy receiver. An accessor write is therefore journaled as its own
|
|
56
|
+
// entry and undone through the same setter, rather than through whatever it writes underneath.
|
|
57
|
+
const proxy = new Proxy(target, {
|
|
58
|
+
get: (object, key) => {
|
|
59
|
+
const value = Reflect.get(object, key);
|
|
60
|
+
return isTrackable(value) ? this.track(value, [...path, key]) : value;
|
|
61
|
+
},
|
|
62
|
+
set: (object, key, rawNext) => {
|
|
63
|
+
const next = unwrap(rawNext);
|
|
64
|
+
const had = Reflect.has(object, key);
|
|
65
|
+
const prev = Reflect.get(object, key);
|
|
66
|
+
this.#recordArrayTruncation(object, key, next);
|
|
67
|
+
if (!Reflect.set(object, key, next))
|
|
68
|
+
return false;
|
|
69
|
+
// An array's `length` is implied by its elements and never undone on its own.
|
|
70
|
+
if (!this.#closed && !(Array.isArray(object) && key === 'length')) {
|
|
71
|
+
entry(key, { had, prev, next, deleted: false });
|
|
72
|
+
}
|
|
73
|
+
return true;
|
|
74
|
+
},
|
|
75
|
+
deleteProperty: (object, key) => {
|
|
76
|
+
const had = Reflect.has(object, key);
|
|
77
|
+
const prev = Reflect.get(object, key);
|
|
78
|
+
if (!Reflect.deleteProperty(object, key))
|
|
79
|
+
return false;
|
|
80
|
+
if (!this.#closed && had)
|
|
81
|
+
entry(key, { had, prev, next: undefined, deleted: true });
|
|
82
|
+
return true;
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
this.#proxies.set(target, proxy);
|
|
86
|
+
proxyTargets.set(proxy, target);
|
|
87
|
+
return proxy;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Shrinking an array's `length` drops elements without a trap firing for them. They are recorded as
|
|
91
|
+
* deletions, from the highest index down, so the reverse replay reinserts them from the lowest up.
|
|
92
|
+
*/
|
|
93
|
+
#recordArrayTruncation(object, key, next) {
|
|
94
|
+
if (this.#closed || !Array.isArray(object) || key !== 'length')
|
|
95
|
+
return;
|
|
96
|
+
const path = this.#paths.get(object);
|
|
97
|
+
for (let i = object.length - 1; i >= next; i--) {
|
|
98
|
+
if (i in object) {
|
|
99
|
+
this.#entries.push({
|
|
100
|
+
target: object,
|
|
101
|
+
path,
|
|
102
|
+
key: String(i),
|
|
103
|
+
had: true,
|
|
104
|
+
prev: object[i],
|
|
105
|
+
next: undefined,
|
|
106
|
+
deleted: true,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Restores one entry on `target`. Array elements are handled positionally - an added element is removed
|
|
113
|
+
* and a removed one reinserted, shifting the rest - so the result stays something a sequence of `push`
|
|
114
|
+
* and `splice` calls could have produced, even next to elements another writer added meanwhile.
|
|
115
|
+
*/
|
|
116
|
+
static #undo(target, { key, had, deleted }, value) {
|
|
117
|
+
if (Array.isArray(target) && typeof key === 'string' && Number.isInteger(Number(key))) {
|
|
118
|
+
if (deleted) {
|
|
119
|
+
target.splice(Number(key), 0, value);
|
|
120
|
+
}
|
|
121
|
+
else if (had) {
|
|
122
|
+
target[Number(key)] = value;
|
|
123
|
+
}
|
|
124
|
+
else {
|
|
125
|
+
target.splice(Number(key), 1);
|
|
126
|
+
}
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
if (had) {
|
|
130
|
+
Reflect.set(target, key, value);
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
Reflect.deleteProperty(target, key);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/** Stops recording. Proxies keep writing through. */
|
|
137
|
+
close() {
|
|
138
|
+
this.#closed = true;
|
|
139
|
+
}
|
|
140
|
+
/** Undoes the recorded writes, newest first, and stops recording. */
|
|
141
|
+
revert({ onConflict } = {}) {
|
|
142
|
+
this.close();
|
|
143
|
+
for (let i = this.#entries.length - 1; i >= 0; i--) {
|
|
144
|
+
const entry = this.#entries[i];
|
|
145
|
+
const { target, key, next, deleted } = entry;
|
|
146
|
+
const untouched = deleted ? !Reflect.has(target, key) : Reflect.get(target, key) === next;
|
|
147
|
+
if (!untouched) {
|
|
148
|
+
onConflict?.(key);
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
MutationJournal.#undo(target, entry, entry.prev);
|
|
152
|
+
}
|
|
153
|
+
this.#entries.length = 0;
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Applies the inverse of the recorded writes onto `root`, a JSON copy of the tracked object, and returns
|
|
157
|
+
* it. The journal is left as it is. Restored values are JSON-cloned too, so the copy shares nothing with
|
|
158
|
+
* the original and non-JSON values drop out the same way on both.
|
|
159
|
+
*
|
|
160
|
+
* Entries are located by the path under which their object was *first* reached, so an object reachable
|
|
161
|
+
* through two paths is undone under the first one only.
|
|
162
|
+
*/
|
|
163
|
+
undoOnto(root) {
|
|
164
|
+
for (let i = this.#entries.length - 1; i >= 0; i--) {
|
|
165
|
+
const entry = this.#entries[i];
|
|
166
|
+
const target = entry.path.reduce((object, step) => object?.[step], root);
|
|
167
|
+
if (typeof target !== 'object' || target === null)
|
|
168
|
+
continue;
|
|
169
|
+
MutationJournal.#undo(target, entry, isTrackable(entry.prev) ? JSON.parse(JSON.stringify(entry.prev)) : entry.prev);
|
|
170
|
+
}
|
|
171
|
+
return root;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
@@ -14,7 +14,6 @@ type OpenGraphResult = string | string[] | Dictionary<string | Dictionary>;
|
|
|
14
14
|
* Currently existing properties are kept up to date.
|
|
15
15
|
* @returns Scraped OpenGraph properties as an object.
|
|
16
16
|
*/
|
|
17
|
-
export declare function parseOpenGraph(raw: string, additionalProperties?: OpenGraphProperty[]): Dictionary<OpenGraphResult
|
|
18
|
-
export declare function parseOpenGraph($: CheerioAPI, additionalProperties?: OpenGraphProperty[]): Dictionary<OpenGraphResult
|
|
17
|
+
export declare function parseOpenGraph(raw: string, additionalProperties?: OpenGraphProperty[]): Promise<Dictionary<OpenGraphResult>>;
|
|
18
|
+
export declare function parseOpenGraph($: CheerioAPI, additionalProperties?: OpenGraphProperty[]): Promise<Dictionary<OpenGraphResult>>;
|
|
19
19
|
export {};
|
|
20
|
-
//# sourceMappingURL=open_graph_parser.d.ts.map
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { load } from 'cheerio';
|
|
2
1
|
/**
|
|
3
2
|
* To be used with the spread operator. Ensures that the item is defined, and is not empty.
|
|
4
3
|
*
|
|
@@ -255,32 +254,32 @@ const OPEN_GRAPH_PROPERTIES = [
|
|
|
255
254
|
outputName: 'articleInfo',
|
|
256
255
|
children: [
|
|
257
256
|
{
|
|
258
|
-
name: '
|
|
257
|
+
name: 'article:published_time',
|
|
259
258
|
outputName: 'publishedTime',
|
|
260
259
|
children: [],
|
|
261
260
|
},
|
|
262
261
|
{
|
|
263
|
-
name: '
|
|
262
|
+
name: 'article:modified_time',
|
|
264
263
|
outputName: 'modifiedTime',
|
|
265
264
|
children: [],
|
|
266
265
|
},
|
|
267
266
|
{
|
|
268
|
-
name: '
|
|
267
|
+
name: 'article:expiration_time',
|
|
269
268
|
outputName: 'expirationTime',
|
|
270
269
|
children: [],
|
|
271
270
|
},
|
|
272
271
|
{
|
|
273
|
-
name: '
|
|
272
|
+
name: 'article:author',
|
|
274
273
|
outputName: 'author',
|
|
275
274
|
children: [],
|
|
276
275
|
},
|
|
277
276
|
{
|
|
278
|
-
name: '
|
|
277
|
+
name: 'article:section',
|
|
279
278
|
outputName: 'section',
|
|
280
279
|
children: [],
|
|
281
280
|
},
|
|
282
281
|
{
|
|
283
|
-
name: '
|
|
282
|
+
name: 'article:tag',
|
|
284
283
|
outputName: 'tag',
|
|
285
284
|
children: [],
|
|
286
285
|
},
|
|
@@ -366,7 +365,8 @@ const parseOpenGraphProperty = (property, $) => {
|
|
|
366
365
|
}, {}),
|
|
367
366
|
};
|
|
368
367
|
};
|
|
369
|
-
export function parseOpenGraph(item, additionalProperties) {
|
|
368
|
+
export async function parseOpenGraph(item, additionalProperties) {
|
|
369
|
+
const { load } = await import('cheerio/slim');
|
|
370
370
|
const $ = typeof item === 'string' ? load(item) : item;
|
|
371
371
|
return [...(additionalProperties || []), ...OPEN_GRAPH_PROPERTIES].reduce((acc, curr) => {
|
|
372
372
|
return {
|
|
@@ -375,4 +375,3 @@ export function parseOpenGraph(item, additionalProperties) {
|
|
|
375
375
|
};
|
|
376
376
|
}, {});
|
|
377
377
|
}
|
|
378
|
-
//# sourceMappingURL=open_graph_parser.js.map
|