@crawlee/utils 4.0.0-beta.241 → 4.0.0-beta.242

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/internal.d.ts CHANGED
@@ -4,6 +4,7 @@ export { extractUrlsFromCheerio } from './internals/cheerio.js';
4
4
  export { tryAbsoluteURL } from './internals/extract-urls.js';
5
5
  export { expandShadowRoots, URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
6
6
  export * from './internals/iterables.js';
7
+ export { MutationJournal } from './internals/mutation-journal.js';
7
8
  export * from './internals/url.js';
8
9
  export * from './internals/validation.js';
9
10
  export * as schemas from './internals/schemas.js';
package/internal.js CHANGED
@@ -3,6 +3,7 @@ export { extractUrlsFromCheerio } from './internals/cheerio.js';
3
3
  export { tryAbsoluteURL } from './internals/extract-urls.js';
4
4
  export { expandShadowRoots, URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
5
5
  export * from './internals/iterables.js';
6
+ export { MutationJournal } from './internals/mutation-journal.js';
6
7
  export * from './internals/url.js';
7
8
  export * from './internals/validation.js';
8
9
  export * as schemas from './internals/schemas.js';
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Records property writes made through the proxies it hands out, applying them to the real objects as
3
+ * they happen, so that a losing attempt's writes can be undone afterwards - on the objects themselves
4
+ * ({@link MutationJournal.revert|`revert()`}), or on a copy of them
5
+ * ({@link MutationJournal.undoOnto|`undoOnto()`}) to reconstruct how they looked before the attempt.
6
+ *
7
+ * An undo on the live object only touches a property that still holds the value the attempt wrote - a
8
+ * value written by someone else in the meantime is left alone and reported via `onConflict`. Writes that
9
+ * bypass the proxy are invisible until they trip that guard.
10
+ */
11
+ export declare class MutationJournal {
12
+ #private;
13
+ track<T extends object>(target: T, path?: PropertyKey[]): T;
14
+ /** Stops recording. Proxies keep writing through. */
15
+ close(): void;
16
+ /** Undoes the recorded writes, newest first, and stops recording. */
17
+ revert({ onConflict }?: {
18
+ onConflict?: (key: PropertyKey) => void;
19
+ }): void;
20
+ /**
21
+ * Applies the inverse of the recorded writes onto `root`, a JSON copy of the tracked object, and returns
22
+ * it. The journal is left as it is. Restored values are JSON-cloned too, so the copy shares nothing with
23
+ * the original and non-JSON values drop out the same way on both.
24
+ *
25
+ * Entries are located by the path under which their object was *first* reached, so an object reachable
26
+ * through two paths is undone under the first one only.
27
+ */
28
+ undoOnto<T extends object>(root: T): T;
29
+ }
@@ -0,0 +1,173 @@
1
+ /**
2
+ * Only plain objects and arrays are wrapped when reached through a tracked object. Built-ins such as `Map`
3
+ * or `Set` reject a proxy receiver in their methods, and the tracked objects are JSON-shaped by contract
4
+ * anyway - such values are handed out as they are, and their mutations are not recorded.
5
+ */
6
+ const isTrackable = (value) => Array.isArray(value) ||
7
+ (typeof value === 'object' &&
8
+ value !== null &&
9
+ (Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null));
10
+ /** Every proxy any journal has handed out, keyed by the proxy. Shared, as a value read through one attempt's
11
+ * proxy may be written back through another's. */
12
+ const proxyTargets = new WeakMap();
13
+ /**
14
+ * Replaces journal proxies in `value` with the objects behind them, in place for nested ones. A value read
15
+ * through a proxy and written back (`state.x = state.x ?? {}`) would otherwise store the proxy itself, and
16
+ * every later attempt would wrap it once more.
17
+ */
18
+ function unwrap(value, seen = new Set()) {
19
+ const target = typeof value === 'object' && value !== null ? proxyTargets.get(value) : undefined;
20
+ if (target)
21
+ return target;
22
+ if (!isTrackable(value) || seen.has(value))
23
+ return value;
24
+ seen.add(value);
25
+ for (const key of Object.keys(value)) {
26
+ const inner = value[key];
27
+ const unwrapped = unwrap(inner, seen);
28
+ if (unwrapped !== inner)
29
+ value[key] = unwrapped;
30
+ }
31
+ return value;
32
+ }
33
+ /**
34
+ * Records property writes made through the proxies it hands out, applying them to the real objects as
35
+ * they happen, so that a losing attempt's writes can be undone afterwards - on the objects themselves
36
+ * ({@link MutationJournal.revert|`revert()`}), or on a copy of them
37
+ * ({@link MutationJournal.undoOnto|`undoOnto()`}) to reconstruct how they looked before the attempt.
38
+ *
39
+ * An undo on the live object only touches a property that still holds the value the attempt wrote - a
40
+ * value written by someone else in the meantime is left alone and reported via `onConflict`. Writes that
41
+ * bypass the proxy are invisible until they trip that guard.
42
+ */
43
+ export class MutationJournal {
44
+ #entries = [];
45
+ #proxies = new WeakMap();
46
+ #paths = new WeakMap();
47
+ #closed = false;
48
+ track(target, path = []) {
49
+ const cached = this.#proxies.get(target);
50
+ if (cached)
51
+ return cached;
52
+ this.#paths.set(target, path);
53
+ const entry = (key, fields) => this.#entries.push({ target, path, key, ...fields });
54
+ // Accessors run with `this` as the real object, not the proxy: class accessors may touch `#private`
55
+ // members, which throw on a proxy receiver. An accessor write is therefore journaled as its own
56
+ // entry and undone through the same setter, rather than through whatever it writes underneath.
57
+ const proxy = new Proxy(target, {
58
+ get: (object, key) => {
59
+ const value = Reflect.get(object, key);
60
+ return isTrackable(value) ? this.track(value, [...path, key]) : value;
61
+ },
62
+ set: (object, key, rawNext) => {
63
+ const next = unwrap(rawNext);
64
+ const had = Reflect.has(object, key);
65
+ const prev = Reflect.get(object, key);
66
+ this.#recordArrayTruncation(object, key, next);
67
+ if (!Reflect.set(object, key, next))
68
+ return false;
69
+ // An array's `length` is implied by its elements and never undone on its own.
70
+ if (!this.#closed && !(Array.isArray(object) && key === 'length')) {
71
+ entry(key, { had, prev, next, deleted: false });
72
+ }
73
+ return true;
74
+ },
75
+ deleteProperty: (object, key) => {
76
+ const had = Reflect.has(object, key);
77
+ const prev = Reflect.get(object, key);
78
+ if (!Reflect.deleteProperty(object, key))
79
+ return false;
80
+ if (!this.#closed && had)
81
+ entry(key, { had, prev, next: undefined, deleted: true });
82
+ return true;
83
+ },
84
+ });
85
+ this.#proxies.set(target, proxy);
86
+ proxyTargets.set(proxy, target);
87
+ return proxy;
88
+ }
89
+ /**
90
+ * Shrinking an array's `length` drops elements without a trap firing for them. They are recorded as
91
+ * deletions, from the highest index down, so the reverse replay reinserts them from the lowest up.
92
+ */
93
+ #recordArrayTruncation(object, key, next) {
94
+ if (this.#closed || !Array.isArray(object) || key !== 'length')
95
+ return;
96
+ const path = this.#paths.get(object);
97
+ for (let i = object.length - 1; i >= next; i--) {
98
+ if (i in object) {
99
+ this.#entries.push({
100
+ target: object,
101
+ path,
102
+ key: String(i),
103
+ had: true,
104
+ prev: object[i],
105
+ next: undefined,
106
+ deleted: true,
107
+ });
108
+ }
109
+ }
110
+ }
111
+ /**
112
+ * Restores one entry on `target`. Array elements are handled positionally - an added element is removed
113
+ * and a removed one reinserted, shifting the rest - so the result stays something a sequence of `push`
114
+ * and `splice` calls could have produced, even next to elements another writer added meanwhile.
115
+ */
116
+ static #undo(target, { key, had, deleted }, value) {
117
+ if (Array.isArray(target) && typeof key === 'string' && Number.isInteger(Number(key))) {
118
+ if (deleted) {
119
+ target.splice(Number(key), 0, value);
120
+ }
121
+ else if (had) {
122
+ target[Number(key)] = value;
123
+ }
124
+ else {
125
+ target.splice(Number(key), 1);
126
+ }
127
+ return;
128
+ }
129
+ if (had) {
130
+ Reflect.set(target, key, value);
131
+ }
132
+ else {
133
+ Reflect.deleteProperty(target, key);
134
+ }
135
+ }
136
+ /** Stops recording. Proxies keep writing through. */
137
+ close() {
138
+ this.#closed = true;
139
+ }
140
+ /** Undoes the recorded writes, newest first, and stops recording. */
141
+ revert({ onConflict } = {}) {
142
+ this.close();
143
+ for (let i = this.#entries.length - 1; i >= 0; i--) {
144
+ const entry = this.#entries[i];
145
+ const { target, key, next, deleted } = entry;
146
+ const untouched = deleted ? !Reflect.has(target, key) : Reflect.get(target, key) === next;
147
+ if (!untouched) {
148
+ onConflict?.(key);
149
+ continue;
150
+ }
151
+ MutationJournal.#undo(target, entry, entry.prev);
152
+ }
153
+ this.#entries.length = 0;
154
+ }
155
+ /**
156
+ * Applies the inverse of the recorded writes onto `root`, a JSON copy of the tracked object, and returns
157
+ * it. The journal is left as it is. Restored values are JSON-cloned too, so the copy shares nothing with
158
+ * the original and non-JSON values drop out the same way on both.
159
+ *
160
+ * Entries are located by the path under which their object was *first* reached, so an object reachable
161
+ * through two paths is undone under the first one only.
162
+ */
163
+ undoOnto(root) {
164
+ for (let i = this.#entries.length - 1; i >= 0; i--) {
165
+ const entry = this.#entries[i];
166
+ const target = entry.path.reduce((object, step) => object?.[step], root);
167
+ if (typeof target !== 'object' || target === null)
168
+ continue;
169
+ MutationJournal.#undo(target, entry, isTrackable(entry.prev) ? JSON.parse(JSON.stringify(entry.prev)) : entry.prev);
170
+ }
171
+ return root;
172
+ }
173
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/utils",
3
- "version": "4.0.0-beta.241",
3
+ "version": "4.0.0-beta.242",
4
4
  "description": "A set of shared utilities that can be used by crawlers",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -43,8 +43,8 @@
43
43
  },
44
44
  "dependencies": {
45
45
  "@apify/validations": "^1.1.0",
46
- "@crawlee/http-client": "4.0.0-beta.241",
47
- "@crawlee/types": "4.0.0-beta.241",
46
+ "@crawlee/http-client": "4.0.0-beta.242",
47
+ "@crawlee/types": "4.0.0-beta.242",
48
48
  "@types/sax": "^1.2.7",
49
49
  "cheerio": "^1.0.0",
50
50
  "domhandler": "^6.0.0",
@@ -63,5 +63,5 @@
63
63
  }
64
64
  }
65
65
  },
66
- "gitHead": "1c337d1ef50c3e28e25e2ae40bc38bb6de67c37e"
66
+ "gitHead": "9bd381d862366b6b874938fb0717f21fd430b9de"
67
67
  }