@zukhruf/snapshot-differential 0.3.52 → 0.3.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -9
- package/dist/bytes.d.ts +4 -0
- package/dist/bytes.d.ts.map +1 -0
- package/dist/bytes.js +16 -0
- package/dist/bytes.js.map +1 -0
- package/dist/deletion-policy.d.ts +1 -1
- package/dist/deletion-policy.d.ts.map +1 -1
- package/dist/fingerprint.d.ts.map +1 -1
- package/dist/fingerprint.js +23 -13
- package/dist/fingerprint.js.map +1 -1
- package/dist/snapshot-differential.d.ts +5 -2
- package/dist/snapshot-differential.d.ts.map +1 -1
- package/dist/snapshot-differential.js +28 -11
- package/dist/snapshot-differential.js.map +1 -1
- package/dist/snapshot-state.d.ts +4 -1
- package/dist/snapshot-state.d.ts.map +1 -1
- package/dist/snapshot-state.js +11 -1
- package/dist/snapshot-state.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@ The words in these documents have one meaning each. See the glossary in [CONTEXT
|
|
|
6
6
|
|
|
7
7
|
## A diff yields the changes, then the snapshot
|
|
8
8
|
|
|
9
|
-
Make one `SnapshotDifferential` for each scan. Give it the primary key fields, a function that
|
|
9
|
+
Make one `SnapshotDifferential` for each scan. Give it the primary key fields, a function that gives the primary key values of a record in the same order, and a deletion policy. Then give `diff` the scan and the snapshot that the last committed scan saved:
|
|
10
10
|
|
|
11
11
|
```ts
|
|
12
12
|
import {
|
|
@@ -19,18 +19,21 @@ type Contact = { id: string; name: string };
|
|
|
19
19
|
const differential = new SnapshotDifferential<Contact>({
|
|
20
20
|
name: 'contacts',
|
|
21
21
|
primaryKey: ['id'],
|
|
22
|
-
key: (contact) =>
|
|
22
|
+
key: (contact) => [contact.id],
|
|
23
23
|
policy: new ClosedWorld(),
|
|
24
24
|
});
|
|
25
25
|
for await (const change of differential.diff(contacts, saved)) {
|
|
26
|
-
if (change.type === '
|
|
26
|
+
if (change.type === 'insert') await insert(change.record);
|
|
27
|
+
else if (change.type === 'update') await update(change.record);
|
|
27
28
|
else if (change.type === 'delete') await remove(change.key);
|
|
28
29
|
else await save(change.state);
|
|
29
30
|
}
|
|
30
31
|
```
|
|
31
32
|
|
|
32
33
|
- Give `null` as the snapshot for the first scan.
|
|
33
|
-
- The diff yields an
|
|
34
|
+
- The diff yields an insert for each record whose key the snapshot does not have.
|
|
35
|
+
- The diff yields an update for each record whose key the snapshot has with a different fingerprint.
|
|
36
|
+
- The diff yields nothing for a record that did not change.
|
|
34
37
|
- The diff yields the new snapshot last. Save it only after you applied the changes before it.
|
|
35
38
|
- Equal scans give byte-identical snapshots, in any order of records.
|
|
36
39
|
- A failed read must throw. An empty scan makes every key a vanished key.
|
|
@@ -80,7 +83,8 @@ class OpenChannels extends DeletionPolicy {
|
|
|
80
83
|
The policy judges an unknown key again at each scan:
|
|
81
84
|
|
|
82
85
|
- When a later scan covers the key and does not list it, the verdict is `deleted`, and the diff yields a delete.
|
|
83
|
-
- When a later scan lists the record again with the same fingerprint, the diff yields
|
|
86
|
+
- When a later scan lists the record again with the same fingerprint, the diff yields nothing.
|
|
87
|
+
- When a later scan lists the record again with a different fingerprint, the diff yields an update, because the snapshot has the key ([ADR 0005](./docs/adr/0005-a-change-is-an-insert-or-an-update.md)).
|
|
84
88
|
|
|
85
89
|
For example, a chat app drops a thread from its cache. Then a reply in that thread is deleted. Then the app loads the thread again. The reply is not in the thread, and the scan covers it, so the diff deletes it.
|
|
86
90
|
|
|
@@ -106,7 +110,7 @@ for await (const change of differential.diffGroups(
|
|
|
106
110
|
- A group with the same fingerprint as in the snapshot keeps its records. The diff does not call its `records`.
|
|
107
111
|
- A group with a `null` fingerprint is always read.
|
|
108
112
|
- A kept group, `{ key, kept: true }`, keeps its saved fingerprint and records. Give it when you know that the input did not change but you cannot fingerprint it, such as a page that a crawl did not request. A kept group that the snapshot does not have is an error ([ADR 0004](./docs/adr/0004-a-scan-can-keep-a-group-without-reading-it.md)).
|
|
109
|
-
- A record that moves to a different group, with the same fingerprint, gets no
|
|
113
|
+
- A record that moves to a different group, with the same fingerprint, gets no change. With a different fingerprint, it gets an update.
|
|
110
114
|
- The policy judges the vanished keys of the complete scan, also the keys of groups that are not in the scan.
|
|
111
115
|
|
|
112
116
|
## A lookback window keeps only what the next scan reads again
|
|
@@ -133,7 +137,7 @@ for await (const page of profile.pages(after)) {
|
|
|
133
137
|
}
|
|
134
138
|
```
|
|
135
139
|
|
|
136
|
-
- A part yields an
|
|
140
|
+
- A part yields an insert or an update for each new or changed record that it lists. It does not wait for the last part.
|
|
137
141
|
- A part that continues the scan judges no vanished key. Its snapshot keeps every key, and the keys that the open scan listed so far.
|
|
138
142
|
- The last part judges each saved key that no part of the scan listed.
|
|
139
143
|
- In `diffGroups`, a record that leaves a group that a part reads again waits in the snapshot for the last part's judgment.
|
|
@@ -141,13 +145,36 @@ for await (const page of profile.pages(after)) {
|
|
|
141
145
|
- The parts can be in different runs. Save the snapshot after each part, and give it to the next part.
|
|
142
146
|
- The next scan starts with no listed keys.
|
|
143
147
|
|
|
148
|
+
## A record and a key can hold bytes
|
|
149
|
+
|
|
150
|
+
A `Uint8Array` is bytes, and a `Buffer` is a `Uint8Array`. A `BLOB` column of SQLite gives bytes. A record can hold bytes at any depth, and a key value can be bytes:
|
|
151
|
+
|
|
152
|
+
```ts
|
|
153
|
+
type Attachment = { id: Uint8Array; body: Uint8Array };
|
|
154
|
+
|
|
155
|
+
const attachments = new SnapshotDifferential<Attachment>({
|
|
156
|
+
name: 'attachments',
|
|
157
|
+
primaryKey: ['id'],
|
|
158
|
+
key: (attachment) => [attachment.id],
|
|
159
|
+
policy: new ClosedWorld(),
|
|
160
|
+
});
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
- Equal bytes give equal fingerprints, in a `Buffer` or in a `Uint8Array`, and also in a view of a larger buffer.
|
|
164
|
+
- Bytes and a string or an array of numbers with the same content give different fingerprints.
|
|
165
|
+
- The snapshot keeps a key value of bytes as `{"bytes": "<base64url>"}`. No other key value is an object, so no string key has this form.
|
|
166
|
+
- A delete gives a key value of bytes back as a `Uint8Array`.
|
|
167
|
+
- Other typed arrays, such as `Uint16Array`, are not bytes. A record that holds one fails.
|
|
168
|
+
|
|
169
|
+
See [ADR 0006](./docs/adr/0006-a-record-and-a-key-can-hold-bytes.md).
|
|
170
|
+
|
|
144
171
|
## When something fails
|
|
145
172
|
|
|
146
173
|
Each failure is a `TypeError`, and the diff yields no snapshot:
|
|
147
174
|
|
|
148
175
|
- One call lists one key twice: `Snapshot <name> returned <key> twice in one scan`. In a scan in parts, this applies to each part.
|
|
149
176
|
- `diffGroups` gets one group key twice: `Snapshot <name> returned group <key> twice in one scan`.
|
|
150
|
-
- A record has a value that JSON cannot hold without a change, such as `undefined` or a `
|
|
177
|
+
- A record has a value that JSON cannot hold without a change and that is not bytes, such as `undefined`, a `Date` or a `Uint16Array`.
|
|
151
178
|
- A record has no timestamp in its `expiresBy` field, in the format of `Date#toISOString`.
|
|
152
179
|
- The saved state is not a snapshot that this package wrote.
|
|
153
180
|
- `Retention` gets a horizon that is not a timestamp.
|
|
@@ -173,6 +200,8 @@ Airbyte has no equivalent. Outside change data capture, it deletes only when it
|
|
|
173
200
|
- [ADR 0002: A scan can be listed in parts](./docs/adr/0002-a-scan-can-be-listed-in-parts.md)
|
|
174
201
|
- [ADR 0003: The differential is a zukhruf package](./docs/adr/0003-the-differential-is-a-zukhruf-package.md)
|
|
175
202
|
- [ADR 0004: A scan can keep a group without reading it](./docs/adr/0004-a-scan-can-keep-a-group-without-reading-it.md)
|
|
203
|
+
- [ADR 0005: A change is an insert or an update](./docs/adr/0005-a-change-is-an-insert-or-an-update.md)
|
|
204
|
+
- [ADR 0006: A record and a key can hold bytes](./docs/adr/0006-a-record-and-a-key-can-hold-bytes.md)
|
|
176
205
|
|
|
177
206
|
## Development
|
|
178
207
|
|
|
@@ -191,7 +220,7 @@ src/
|
|
|
191
220
|
closed-world.ts ClosedWorld
|
|
192
221
|
retention.ts Retention
|
|
193
222
|
coverage.ts Coverage
|
|
194
|
-
fingerprint.ts fingerprints over canonical JSON
|
|
223
|
+
fingerprint.ts fingerprints over canonical JSON, with bytes
|
|
195
224
|
timestamp.ts the timestamp check
|
|
196
225
|
index.ts the public exports
|
|
197
226
|
snapshot-differential.test.ts the tests
|
package/dist/bytes.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bytes.d.ts","sourceRoot":"","sources":["../src/bytes.ts"],"names":[],"mappings":"AAQA,wBAAgB,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAEnD;AAED,wBAAgB,GAAG,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAE7C;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CAEtD"}
|
package/dist/bytes.js
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
// Bytes as text through Buffer, which every Node.js LTS has: the
|
|
3
|
+
// Uint8Array.prototype.toBase64 and toHex methods are not in Node.js 24.
|
|
4
|
+
function view(bytes) {
|
|
5
|
+
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
6
|
+
}
|
|
7
|
+
export function base64url(bytes) {
|
|
8
|
+
return view(bytes).toString('base64url');
|
|
9
|
+
}
|
|
10
|
+
export function hex(bytes) {
|
|
11
|
+
return view(bytes).toString('hex');
|
|
12
|
+
}
|
|
13
|
+
export function fromBase64url(text) {
|
|
14
|
+
return new Uint8Array(Buffer.from(text, 'base64url'));
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=bytes.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bytes.js","sourceRoot":"","sources":["../src/bytes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,iEAAiE;AACjE,yEAAyE;AACzE,SAAS,IAAI,CAAC,KAAiB;IAC7B,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAiB;IACzC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,KAAiB;IACnC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;AACxD,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"deletion-policy.d.ts","sourceRoot":"","sources":["../src/deletion-policy.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"deletion-policy.d.ts","sourceRoot":"","sources":["../src/deletion-policy.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,UAAU,CAAC;AAG9D,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IAEjD,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;AAOF,MAAM,MAAM,OAAO,GAAG,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;AAGxD,8BAAsB,cAAc;IAClC,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO;CACnD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fingerprint.d.ts","sourceRoot":"","sources":["../src/fingerprint.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fingerprint.d.ts","sourceRoot":"","sources":["../src/fingerprint.ts"],"names":[],"mappings":"AAKA,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B,MAAM,CAIR;AA4BD,wBAAgB,aAAa,CAC3B,KAAK,EAAE,OAAO,GACb,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAOlC"}
|
package/dist/fingerprint.js
CHANGED
|
@@ -1,22 +1,32 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
-
import {
|
|
3
|
-
// A record's SHA-256 over its canonical
|
|
2
|
+
import { base64url } from "./bytes.js";
|
|
3
|
+
// A record's SHA-256 over its canonical form, so equal records hash equally.
|
|
4
4
|
export function fingerprintOf(name, record) {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
return createHash('sha256').update(canonical(serialized)).digest('base64url');
|
|
5
|
+
return createHash('sha256')
|
|
6
|
+
.update(canonical(name, record))
|
|
7
|
+
.digest('base64url');
|
|
9
8
|
}
|
|
10
|
-
// JSON with object keys sorted at every depth.
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
if (
|
|
9
|
+
// JSON with object keys sorted at every depth. Bytes are # and their
|
|
10
|
+
// base64url: a JSON text has no # outside a string, so no JSON value has the
|
|
11
|
+
// form of bytes. A value that JSON cannot hold with no change fails.
|
|
12
|
+
function canonical(name, value) {
|
|
13
|
+
if (value === null ||
|
|
14
|
+
typeof value === 'string' ||
|
|
15
|
+
typeof value === 'boolean' ||
|
|
16
|
+
(typeof value === 'number' &&
|
|
17
|
+
Number.isFinite(value) &&
|
|
18
|
+
!Object.is(value, -0)))
|
|
19
|
+
return JSON.stringify(value);
|
|
20
|
+
if (value instanceof Uint8Array)
|
|
21
|
+
return `#${base64url(value)}`;
|
|
22
|
+
if (Array.isArray(value) && Object.keys(value).length === value.length)
|
|
23
|
+
return `[${value.map((item) => canonical(name, item)).join(',')}]`;
|
|
24
|
+
if (isPlainObject(value) && Object.getOwnPropertySymbols(value).length === 0)
|
|
15
25
|
return `{${Object.keys(value)
|
|
16
26
|
.sort()
|
|
17
|
-
.map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`)
|
|
27
|
+
.map((key) => `${JSON.stringify(key)}:${canonical(name, value[key])}`)
|
|
18
28
|
.join(',')}}`;
|
|
19
|
-
|
|
29
|
+
throw new TypeError(`Snapshot ${name} records must be losslessly JSON serializable, or bytes, to diff snapshots`);
|
|
20
30
|
}
|
|
21
31
|
export function isPlainObject(value) {
|
|
22
32
|
return (value !== null &&
|
package/dist/fingerprint.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fingerprint.js","sourceRoot":"","sources":["../src/fingerprint.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"fingerprint.js","sourceRoot":"","sources":["../src/fingerprint.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,6EAA6E;AAC7E,MAAM,UAAU,aAAa,CAC3B,IAAY,EACZ,MAA+B;IAE/B,OAAO,UAAU,CAAC,QAAQ,CAAC;SACxB,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;SAC/B,MAAM,CAAC,WAAW,CAAC,CAAC;AACzB,CAAC;AAED,qEAAqE;AACrE,6EAA6E;AAC7E,qEAAqE;AACrE,SAAS,SAAS,CAAC,IAAY,EAAE,KAAc;IAC7C,IACE,KAAK,KAAK,IAAI;QACd,OAAO,KAAK,KAAK,QAAQ;QACzB,OAAO,KAAK,KAAK,SAAS;QAC1B,CAAC,OAAO,KAAK,KAAK,QAAQ;YACxB,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YACtB,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QAExB,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;IAC/D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;QACpE,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACrE,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC;QAC1E,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;aAC1B,IAAI,EAAE;aACN,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;aACrE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IAClB,MAAM,IAAI,SAAS,CACjB,YAAY,IAAI,4EAA4E,CAC7F,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,KAAc;IAEd,OAAO,CACL,KAAK,KAAK,IAAI;QACd,OAAO,KAAK,KAAK,QAAQ;QACzB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACrB,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC,SAAS,CAClD,CAAC;AACJ,CAAC"}
|
|
@@ -3,7 +3,7 @@ import { type GroupedSnapshotState, type SnapshotState } from './snapshot-state.
|
|
|
3
3
|
export type SnapshotDifferentialOptions<Data> = {
|
|
4
4
|
readonly name: string;
|
|
5
5
|
readonly primaryKey: readonly string[];
|
|
6
|
-
readonly key: (record: Data) =>
|
|
6
|
+
readonly key: (record: Data) => readonly KeyValue[];
|
|
7
7
|
readonly expiresBy?: string | undefined;
|
|
8
8
|
readonly policy: DeletionPolicy;
|
|
9
9
|
};
|
|
@@ -17,7 +17,10 @@ export type KeptGroup = {
|
|
|
17
17
|
readonly kept: true;
|
|
18
18
|
};
|
|
19
19
|
export type SnapshotChange<Data, State> = {
|
|
20
|
-
readonly type: '
|
|
20
|
+
readonly type: 'insert';
|
|
21
|
+
readonly record: Data;
|
|
22
|
+
} | {
|
|
23
|
+
readonly type: 'update';
|
|
21
24
|
readonly record: Data;
|
|
22
25
|
} | {
|
|
23
26
|
readonly type: 'delete';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot-differential.d.ts","sourceRoot":"","sources":["../src/snapshot-differential.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"snapshot-differential.d.ts","sourceRoot":"","sources":["../src/snapshot-differential.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,cAAc,EACd,QAAQ,EAET,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,KAAK,oBAAoB,EAGzB,KAAK,aAAa,EAOnB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,MAAM,2BAA2B,CAAC,IAAI,IAAI;IAE9C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IAEvC,QAAQ,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,IAAI,KAAK,SAAS,QAAQ,EAAE,CAAC;IAGpD,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;CACjC,CAAC;AAIF,MAAM,MAAM,aAAa,CAAC,IAAI,IAAI;IAChC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAErB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,OAAO,IAAI,aAAa,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;CACjD,CAAC;AAIF,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,IAClC;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAA;CAAE,GAClD;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAA;CAAE,GAClD;IACE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;CAClD,GACD;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAA;CAAE,CAAC;AAStD,qBAAa,oBAAoB,CAAC,IAAI,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;;IACpE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;gBAGpB,EACV,IAAI,EACJ,UAAU,EACV,GAAG,EACH,SAAS,EACT,MAAM,GACP,EAAE,2BAA2B,CAAC,IAAI,CAAC;IAe7B,IAAI,CACT,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,EAC7C,KAAK,EAAE,OAAO,EACd,EACE,OAAO,EACP,SAAiB,GAClB,GAAE;QACD,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,KAAK,OAAO,CAAC;QAC7C,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;KACzB,GACL,cAAc,CAAC,cAAc,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAsC/C,UAAU,CACf,MAAM,EACF,aAAa,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,GAC9C,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,EAC7C,KAAK,EAAE,OAAO,EACd,EAAE,SAAiB,EAAE,GAAE;QAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAO,GAC3D,cAAc,CAAC,cAAc,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;CAqJ9D"}
|
|
@@ -1,10 +1,13 @@
|
|
|
1
|
+
import { hex } from "./bytes.js";
|
|
1
2
|
import { fingerprintOf } from "./fingerprint.js";
|
|
2
|
-
import { fingerprintIn, readGroups, readSnapshot, sorted, } from "./snapshot-state.js";
|
|
3
|
+
import { fingerprintIn, keyOf, keyValuesOf, readGroups, readSnapshot, sorted, } from "./snapshot-state.js";
|
|
3
4
|
import { isTimestamp } from "./timestamp.js";
|
|
4
5
|
// A snapshot differential (Labio and Garcia-Molina, VLDB 1996): change data
|
|
5
6
|
// capture for an upstream with no change log. It compares one complete scan
|
|
6
|
-
// with the previous snapshot and yields
|
|
7
|
-
//
|
|
7
|
+
// with the previous snapshot and yields an insert for each record whose key
|
|
8
|
+
// the snapshot does not have, an update for each record whose key it has with
|
|
9
|
+
// another fingerprint, a delete for each key the policy judges deleted, and
|
|
10
|
+
// the new snapshot last. An empty
|
|
8
11
|
// scan judges every key, so a failed read must throw, never yield nothing.
|
|
9
12
|
export class SnapshotDifferential {
|
|
10
13
|
name;
|
|
@@ -35,8 +38,11 @@ export class SnapshotDifferential {
|
|
|
35
38
|
const fingerprint = fingerprintOf(this.name, record);
|
|
36
39
|
if (rereads?.(record) ?? true)
|
|
37
40
|
current.set(key, this.#entry(record, fingerprint));
|
|
38
|
-
|
|
39
|
-
|
|
41
|
+
const before = previous.get(key);
|
|
42
|
+
if (before === undefined)
|
|
43
|
+
yield { type: 'insert', record };
|
|
44
|
+
else if (fingerprintIn(before) !== fingerprint)
|
|
45
|
+
yield { type: 'update', record };
|
|
40
46
|
}
|
|
41
47
|
const listedSoFar = new Set([...listed, ...seen]);
|
|
42
48
|
for (const [key, entry] of previous)
|
|
@@ -92,8 +98,11 @@ export class SnapshotDifferential {
|
|
|
92
98
|
const key = this.#claim(seen, record);
|
|
93
99
|
const fingerprint = fingerprintOf(this.name, record);
|
|
94
100
|
snapshot.set(key, this.#entry(record, fingerprint));
|
|
95
|
-
|
|
96
|
-
|
|
101
|
+
const before = saved.get(key);
|
|
102
|
+
if (before === undefined)
|
|
103
|
+
yield { type: 'insert', record };
|
|
104
|
+
else if (fingerprintIn(before) !== fingerprint)
|
|
105
|
+
yield { type: 'update', record };
|
|
97
106
|
}
|
|
98
107
|
current.set(group.key, {
|
|
99
108
|
fingerprint: group.fingerprint,
|
|
@@ -153,7 +162,7 @@ export class SnapshotDifferential {
|
|
|
153
162
|
}
|
|
154
163
|
}
|
|
155
164
|
#claim(seen, record) {
|
|
156
|
-
const key = this.#key(record);
|
|
165
|
+
const key = keyOf(this.#key(record));
|
|
157
166
|
this.#claimKey(seen, key);
|
|
158
167
|
return key;
|
|
159
168
|
}
|
|
@@ -172,7 +181,7 @@ export class SnapshotDifferential {
|
|
|
172
181
|
}
|
|
173
182
|
// A key as its primary key fields, from the JSON of its values.
|
|
174
183
|
#keyObject(key) {
|
|
175
|
-
const values =
|
|
184
|
+
const values = keyValuesOf(key);
|
|
176
185
|
return Object.fromEntries(this.primaryKey.map((field, index) => {
|
|
177
186
|
const value = values[index];
|
|
178
187
|
if (value === undefined)
|
|
@@ -180,11 +189,19 @@ export class SnapshotDifferential {
|
|
|
180
189
|
return [field, value];
|
|
181
190
|
}));
|
|
182
191
|
}
|
|
183
|
-
// A key as people read it: each field with its value, a string as it is
|
|
192
|
+
// A key as people read it: each field with its value, a string as it is
|
|
193
|
+
// and bytes in hex.
|
|
184
194
|
#keyText(key) {
|
|
185
195
|
return Object.entries(this.#keyObject(key))
|
|
186
|
-
.map(([field, value]) => `${field} ${
|
|
196
|
+
.map(([field, value]) => `${field} ${valueText(value)}`)
|
|
187
197
|
.join(', ');
|
|
188
198
|
}
|
|
189
199
|
}
|
|
200
|
+
function valueText(value) {
|
|
201
|
+
if (typeof value === 'string')
|
|
202
|
+
return value;
|
|
203
|
+
if (value instanceof Uint8Array)
|
|
204
|
+
return `bytes ${hex(value)}`;
|
|
205
|
+
return JSON.stringify(value);
|
|
206
|
+
}
|
|
190
207
|
//# sourceMappingURL=snapshot-differential.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot-differential.js","sourceRoot":"","sources":["../src/snapshot-differential.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"snapshot-differential.js","sourceRoot":"","sources":["../src/snapshot-differential.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAMjC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAKL,aAAa,EACb,KAAK,EACL,WAAW,EACX,UAAU,EACV,YAAY,EACZ,MAAM,GACP,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAuC7C,4EAA4E;AAC5E,4EAA4E;AAC5E,4EAA4E;AAC5E,8EAA8E;AAC9E,4EAA4E;AAC5E,kCAAkC;AAClC,2EAA2E;AAC3E,MAAM,OAAO,oBAAoB;IACtB,IAAI,CAAS;IACb,UAAU,CAAoB;IAC9B,SAAS,CAAqB;IAC9B,MAAM,CAAiB;IACvB,IAAI,CAAwC;IAErD,YAAY,EACV,IAAI,EACJ,UAAU,EACV,GAAG,EACH,SAAS,EACT,MAAM,GAC4B;QAClC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;QAChB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,yEAAyE;IACzE,0EAA0E;IAC1E,wEAAwE;IACxE,sEAAsE;IACtE,sEAAsE;IACtE,yEAAyE;IACzE,4DAA4D;IAC5D,KAAK,CAAC,CAAC,IAAI,CACT,OAA6C,EAC7C,KAAc,EACd,EACE,OAAO,EACP,SAAS,GAAG,KAAK,MAIf,EAAE;QAEN,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;QAC9C,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACtC,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACrD,IAAI,OAAO,EAAE,CAAC,MAAM,CAAC,IAAI,IAAI;gBAC3B,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,MAAM,KAAK,SAAS;gBAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;iBACtD,IAAI,aAAa,CAAC,MAAM,CAAC,KAAK,WAAW;gBAC5C,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QACrC,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QAClD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,QAAQ;YACjC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAClD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC5B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM;gBACJ,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE;oBACL,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC;oBACzB,MAAM,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,IAAI,EAAE;iBAChC;aACF,CAAC;YACF,OAAO;QACT,CAAC;QACD,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;IAChE,CAAC;IAED,yEAAyE;IACzE,4EAA4E;IAC5E,4EAA4E;IAC5E,sEAAsE;IACtE,2EAA2E;IAC3E,YAAY;IACZ,KAAK,CAAC,CAAC,UAAU,CACf,MAE6C,EAC7C,KAAc,EACd,EAAE,SAAS,GAAG,KAAK,KAAuC,EAAE;QAE5D,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC9C,qEAAqE;QACrE,qEAAqE;QACrE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;QAC5C,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE;YACjD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC;gBACjD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC1B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,OAAO;YAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;QAC9C,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACjC,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;gBACxB,MAAM,IAAI,SAAS,CACjB,YAAY,IAAI,CAAC,IAAI,mBAAmB,KAAK,CAAC,GAAG,oBAAoB,CACtE,CAAC;YACJ,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC9C,IACE,MAAM,IAAI,KAAK;gBACf,CAAC,KAAK,CAAC,WAAW,KAAK,IAAI;oBACzB,MAAM,EAAE,WAAW,KAAK,KAAK,CAAC,WAAW,CAAC,EAC5C,CAAC;gBACD,IAAI,MAAM,KAAK,SAAS;oBACtB,MAAM,IAAI,SAAS,CACjB,YAAY,IAAI,CAAC,IAAI,eAAe,KAAK,CAAC,GAAG,oCAAoC,CAClF,CAAC;gBACJ,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;oBAC5C,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC/B,SAAS;YACX,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;YAC/C,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBACtC,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBACrD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;gBACpD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,MAAM,KAAK,SAAS;oBAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;qBACtD,IAAI,aAAa,CAAC,MAAM,CAAC,KAAK,WAAW;oBAC5C,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;YACrC,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE;gBACrB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;aAC3B,CAAC,CAAC;QACL,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACrE,yEAAyE;QACzE,uEAAuE;QACvE,2CAA2C;QAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC5C,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC9D,SAAS;YACX,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CACrD,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CACtC,CAAC;YACF,KAAK,MAAM,CAAC,SAAS,CAAC,IAAI,QAAQ;gBAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACxD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE;gBACf,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC;aACvC,CAAC,CAAC;QACL,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;QAC9C,IAAI,SAAS,EAAE,CAAC;YACd,kEAAkE;YAClE,kEAAkE;YAClE,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CACvE,CAAC;YACF,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK;gBAC9B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM;YACJ,IAAI,EAAE,OAAO;YACb,KAAK,EAAE;gBACL,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC;gBACvB,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3D,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1D;SACF,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,4EAA4E;IAC5E,CAAC,MAAM,CACL,KAAsC,EACtC,IAAyB,EACzB,IAA6B;QAE7B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAC5B,MAAM,MAAM,GACV,OAAO,KAAK,KAAK,QAAQ;gBACvB,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;gBAC/B,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC7C,IAAI,OAAO,KAAK,SAAS;gBAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;iBAChE,IAAI,OAAO,KAAK,SAAS;gBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAiB,EAAE,MAAY;QACpC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACrC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC1B,OAAO,GAAG,CAAC;IACb,CAAC;IAED,SAAS,CAAC,IAAiB,EAAE,GAAW;QACtC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YACf,MAAM,IAAI,SAAS,CACjB,YAAY,IAAI,CAAC,IAAI,aAAa,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,oBAAoB,CACzE,CAAC;QACJ,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,MAAY,EAAE,WAAmB;QACtC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;YAAE,OAAO,WAAW,CAAC;QACrD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC;YACzB,MAAM,IAAI,SAAS,CACjB,YAAY,IAAI,CAAC,IAAI,uBAAuB,IAAI,CAAC,SAAS,2BAA2B,CACtF,CAAC;QACJ,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IAClC,CAAC;IAED,gEAAgE;IAChE,UAAU,CAAC,GAAW;QACpB,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAChC,OAAO,MAAM,CAAC,WAAW,CACvB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,KAAK,KAAK,SAAS;gBACrB,MAAM,IAAI,SAAS,CACjB,gBAAgB,GAAG,mBAAmB,IAAI,CAAC,IAAI,gBAAgB,CAChE,CAAC;YACJ,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACxB,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,wEAAwE;IACxE,oBAAoB;IACpB,QAAQ,CAAC,GAAW;QAClB,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aACxC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;aACvD,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;CACF;AAED,SAAS,SAAS,CAAC,KAAe;IAChC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,SAAS,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;IAC9D,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC"}
|
package/dist/snapshot-state.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { KeyValue } from './deletion-policy.ts';
|
|
1
2
|
export type SavedEntry = string | readonly [fingerprint: string, expiresBy: string];
|
|
2
3
|
export type SnapshotState = {
|
|
3
4
|
readonly snapshot: Readonly<Record<string, SavedEntry>>;
|
|
@@ -22,5 +23,7 @@ export declare function readGroups(name: string, state: unknown): {
|
|
|
22
23
|
listed: Set<string>;
|
|
23
24
|
};
|
|
24
25
|
export declare function sorted<Value>(values: ReadonlyMap<string, Value>): Record<string, Value>;
|
|
25
|
-
export declare function fingerprintIn(entry: SavedEntry
|
|
26
|
+
export declare function fingerprintIn(entry: SavedEntry): string;
|
|
27
|
+
export declare function keyOf(values: readonly KeyValue[]): string;
|
|
28
|
+
export declare function keyValuesOf(key: string): KeyValue[];
|
|
26
29
|
//# sourceMappingURL=snapshot-state.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot-state.d.ts","sourceRoot":"","sources":["../src/snapshot-state.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"snapshot-state.d.ts","sourceRoot":"","sources":["../src/snapshot-state.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAMrD,MAAM,MAAM,UAAU,GACpB,MAAM,GAAG,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;AAK7D,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;CACzD,CAAC;AAMF,MAAM,MAAM,oBAAoB,GAAG;IACjC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IACtD,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACrC,CAAC;AAIF,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,GACb;IAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;CAAE,CAO5D;AAKD,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,GACb;IACD,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAChC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACjC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CACrB,CAwBA;AAGD,wBAAgB,MAAM,CAAC,KAAK,EAC1B,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,GACjC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAEvB;AASD,wBAAgB,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAEvD;AAOD,wBAAgB,KAAK,CAAC,MAAM,EAAE,SAAS,QAAQ,EAAE,GAAG,MAAM,CAMzD;AAGD,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,EAAE,CAKnD"}
|
package/dist/snapshot-state.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { base64url, fromBase64url } from "./bytes.js";
|
|
1
2
|
import { isPlainObject } from "./fingerprint.js";
|
|
2
3
|
// The previous snapshot and the keys the open scan's earlier parts listed,
|
|
3
4
|
// as diff wrote them. null when there is none.
|
|
@@ -49,7 +50,16 @@ function compareKeys(a, b) {
|
|
|
49
50
|
return 0;
|
|
50
51
|
}
|
|
51
52
|
export function fingerprintIn(entry) {
|
|
52
|
-
return typeof entry === 'string' ? entry : entry
|
|
53
|
+
return typeof entry === 'string' ? entry : entry[0];
|
|
54
|
+
}
|
|
55
|
+
// The JSON array of a record's primary key values.
|
|
56
|
+
export function keyOf(values) {
|
|
57
|
+
return JSON.stringify(values.map((value) => value instanceof Uint8Array ? { bytes: base64url(value) } : value));
|
|
58
|
+
}
|
|
59
|
+
// The primary key values of a key that keyOf wrote.
|
|
60
|
+
export function keyValuesOf(key) {
|
|
61
|
+
const values = JSON.parse(key);
|
|
62
|
+
return values.map((value) => typeof value === 'object' ? fromBase64url(value.bytes) : value);
|
|
53
63
|
}
|
|
54
64
|
function entries(name, snapshot) {
|
|
55
65
|
if (!isPlainObject(snapshot))
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot-state.js","sourceRoot":"","sources":["../src/snapshot-state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AA+BjD,2EAA2E;AAC3E,+CAA+C;AAC/C,MAAM,UAAU,YAAY,CAC1B,IAAY,EACZ,KAAc;IAEd,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IACtE,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC;QAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,OAAO;QACL,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC;QACvC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;KACvC,CAAC;AACJ,CAAC;AAED,2EAA2E;AAC3E,wEAAwE;AACxE,0EAA0E;AAC1E,MAAM,UAAU,UAAU,CACxB,IAAY,EACZ,KAAc;IAMd,IAAI,KAAK,KAAK,IAAI;QAChB,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IACtE,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC;QACvD,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACxD,IACE,CAAC,aAAa,CAAC,KAAK,CAAC;YACrB,CAAC,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAC;YACrE,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC;YAE3B,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE;YACd,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC,CAAC;IACL,CAAC;IACD,OAAO;QACL,MAAM;QACN,OAAO,EACL,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC;QACxE,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;KACvC,CAAC;AACJ,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,MAAM,CACpB,MAAkC;IAElC,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/E,CAAC;AAED,4EAA4E;AAC5E,SAAS,WAAW,CAAC,CAAS,EAAE,CAAS;IACvC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IACrB,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,aAAa,
|
|
1
|
+
{"version":3,"file":"snapshot-state.js","sourceRoot":"","sources":["../src/snapshot-state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEtD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AA+BjD,2EAA2E;AAC3E,+CAA+C;AAC/C,MAAM,UAAU,YAAY,CAC1B,IAAY,EACZ,KAAc;IAEd,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IACtE,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC;QAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,OAAO;QACL,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC;QACvC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;KACvC,CAAC;AACJ,CAAC;AAED,2EAA2E;AAC3E,wEAAwE;AACxE,0EAA0E;AAC1E,MAAM,UAAU,UAAU,CACxB,IAAY,EACZ,KAAc;IAMd,IAAI,KAAK,KAAK,IAAI;QAChB,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IACtE,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC;QACvD,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACxD,IACE,CAAC,aAAa,CAAC,KAAK,CAAC;YACrB,CAAC,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAC;YACrE,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC;YAE3B,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE;YACd,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC,CAAC;IACL,CAAC;IACD,OAAO;QACL,MAAM;QACN,OAAO,EACL,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC;QACxE,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;KACvC,CAAC;AACJ,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,MAAM,CACpB,MAAkC;IAElC,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/E,CAAC;AAED,4EAA4E;AAC5E,SAAS,WAAW,CAAC,CAAS,EAAE,CAAS;IACvC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IACrB,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAiB;IAC7C,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACtD,CAAC;AAMD,mDAAmD;AACnD,MAAM,UAAU,KAAK,CAAC,MAA2B;IAC/C,OAAO,IAAI,CAAC,SAAS,CACnB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAiB,EAAE,CAClC,KAAK,YAAY,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAClE,CACF,CAAC;AACJ,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,MAAM,MAAM,GAAoB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAC/D,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,IAAY,EAAE,QAAiB;IAC9C,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC;QAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC5C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;YAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9C,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,aAAa,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;AAC1E,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,MAAe;IAC/C,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC;QAC3E,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACnB,KAAK,CAAC,MAAM,KAAK,CAAC;YAClB,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ;YAC5B,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAChC,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,OAAO,IAAI,SAAS,CAAC,YAAY,IAAI,gCAAgC,CAAC,CAAC;AACzE,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/snapshot-differential",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.54",
|
|
4
4
|
"description": "Change data capture for an upstream with no change log: compare each scan with the last snapshot, get the new and changed records, and let a deletion policy judge each key that the scan no longer lists.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|