@drupflare/spool 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +263 -41
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/entry.js +33 -1
- package/dist/client/register.d.ts +23 -1
- package/dist/client/register.js +53 -7
- package/dist/protocol/config.d.ts +16 -1
- package/dist/protocol/config.js +3 -1
- package/dist/protocol/dataset.d.ts +183 -0
- package/dist/protocol/dataset.js +206 -0
- package/dist/protocol/envelope.d.ts +1 -0
- package/dist/protocol/envelope.js +2 -1
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/index.d.ts +3 -0
- package/dist/protocol/index.js +3 -0
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +46 -0
- package/dist/protocol/provision.js +38 -0
- package/dist/protocol/query.d.ts +66 -5
- package/dist/protocol/query.js +103 -4
- package/dist/protocol/route.d.ts +6 -2
- package/dist/protocol/route.js +31 -3
- package/dist/protocol/write.d.ts +146 -0
- package/dist/protocol/write.js +101 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +11 -1
- package/dist/report/economics.js +19 -1
- package/dist/report/index.d.ts +2 -0
- package/dist/report/index.js +2 -0
- package/dist/report/inputs.d.ts +97 -0
- package/dist/report/inputs.js +133 -0
- package/dist/report/institution.d.ts +188 -1
- package/dist/report/institution.js +146 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +3 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/dataset.d.ts +48 -0
- package/dist/server/dataset.js +308 -0
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +2 -0
- package/dist/server/spool.d.ts +20 -0
- package/dist/server/spool.js +126 -18
- package/dist/server/write.d.ts +27 -0
- package/dist/server/write.js +127 -0
- package/dist/sw/analytics.d.ts +6 -0
- package/dist/sw/analytics.js +14 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.js +78 -9
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/render.d.ts +7 -2
- package/dist/sw/render.js +19 -5
- package/dist/sw/replica.d.ts +12 -3
- package/dist/sw/replica.js +40 -5
- package/dist/sw/runtime.d.ts +43 -2
- package/dist/sw/runtime.js +340 -24
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +107 -0
- package/dist/sw/write.js +442 -0
- package/package.json +3 -2
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { type DatasetResponse, type DatasetRow } from '../protocol/dataset.js';
|
|
2
|
+
import { type PlanTables } from '../protocol/plan.js';
|
|
3
|
+
import type { DatasetMeta, DatasetRowRecord, ReplicaStorage } from './storage.js';
|
|
4
|
+
/** bytes of a row's JSON, the unit `maxBytes` and the storage budget count in */
|
|
5
|
+
export declare const rowBytes: (row: DatasetRow) => number;
|
|
6
|
+
/** a dataset in memory, as a verified load or a commit left it */
|
|
7
|
+
export type Held = {
|
|
8
|
+
meta: DatasetMeta;
|
|
9
|
+
rows: Map<string, DatasetRow>;
|
|
10
|
+
/** the hash of every row, so a patch never hashes a row it replaces or deletes */
|
|
11
|
+
hashes: Map<string, string>;
|
|
12
|
+
};
|
|
13
|
+
/** what the store needs from the replica around it */
|
|
14
|
+
export type DatasetStoreDeps = {
|
|
15
|
+
storage: ReplicaStorage;
|
|
16
|
+
now(): number;
|
|
17
|
+
/** the replica's wipe count; a commit that began under another one is dropped */
|
|
18
|
+
epoch(): number;
|
|
19
|
+
/** the row hash; {@link rowHash}. A test counts calls through it */
|
|
20
|
+
hash?: (key: string, row: DatasetRow) => Promise<string>;
|
|
21
|
+
/** told after every commit which row keys it put and which it removed */
|
|
22
|
+
onChange?(name: string, puts: string[], deletes: string[]): Promise<void> | void;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* One dataset's state while a response or a run of seed pages is applied, before anything is
|
|
26
|
+
* written. A staged seed replaces the held rows; a staged patch changes them. Nothing here
|
|
27
|
+
* touches storage, so a failed or dropped apply leaves the store as it was.
|
|
28
|
+
*/
|
|
29
|
+
export declare class Staging {
|
|
30
|
+
#private;
|
|
31
|
+
readonly name: string;
|
|
32
|
+
/** whether the commit replaces every held row */
|
|
33
|
+
clear: boolean;
|
|
34
|
+
rows: Map<string, DatasetRow>;
|
|
35
|
+
hashes: Map<string, string>;
|
|
36
|
+
digest: string;
|
|
37
|
+
bytes: number;
|
|
38
|
+
constructor(deps: DatasetStoreDeps, name: string, base?: Held);
|
|
39
|
+
/** applies a response; returns whether the result equals the response's proof, null without one */
|
|
40
|
+
add(response: DatasetResponse): Promise<boolean | null>;
|
|
41
|
+
/** the rows a commit writes: all of them for a replacement, the touched ones for a patch */
|
|
42
|
+
writes(): {
|
|
43
|
+
puts: DatasetRowRecord[];
|
|
44
|
+
deletes: string[];
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* The datasets a device holds. Rows live once in memory after a verified load and once in
|
|
49
|
+
* storage, and a response is applied to a {@link Staging} first and committed in one storage
|
|
50
|
+
* transaction. Reads and writes of the held set go through one queue, so a commit and a wipe
|
|
51
|
+
* never interleave.
|
|
52
|
+
*/
|
|
53
|
+
export declare class DatasetStore {
|
|
54
|
+
#private;
|
|
55
|
+
/** datasets found damaged on a load and cleared, since the worker started */
|
|
56
|
+
evicted: number;
|
|
57
|
+
constructor(deps: DatasetStoreDeps);
|
|
58
|
+
/** the meta of every held dataset */
|
|
59
|
+
list(): Promise<DatasetMeta[]>;
|
|
60
|
+
/**
|
|
61
|
+
* Loads one dataset into memory and checks it: the stored row count and the sum of the stored row
|
|
62
|
+
* hashes must equal the meta. A dataset that fails, which is what a partly evicted store looks
|
|
63
|
+
* like, is cleared and reads as absent. No row is hashed.
|
|
64
|
+
*/
|
|
65
|
+
get(name: string): Promise<Held | undefined>;
|
|
66
|
+
/** the held rows in ascending key order, or undefined when `name` is not loaded and held */
|
|
67
|
+
rows(name: string): Promise<DatasetRow[] | undefined>;
|
|
68
|
+
/** the base table and each joined table a plan reads, or undefined when any of them is not held */
|
|
69
|
+
tables(base: string, joins: readonly string[]): Promise<PlanTables | undefined>;
|
|
70
|
+
/** whether `name` is held and in memory */
|
|
71
|
+
loaded(name: string): boolean;
|
|
72
|
+
/** a held row, looked up in memory; undefined unless {@link DatasetStore.get} loaded the dataset */
|
|
73
|
+
row(name: string, key: string): DatasetRow | undefined;
|
|
74
|
+
/** a staging area over the held rows of `name`, or an empty one for a seed */
|
|
75
|
+
stage(name: string, base?: Held): Staging;
|
|
76
|
+
/**
|
|
77
|
+
* Writes a staged apply. `cursor` and `schema` are what the response brought the rows to.
|
|
78
|
+
* Returns false, writing nothing, when the replica was wiped since `epoch` was read. Rejects,
|
|
79
|
+
* with the held rows and the memory copy untouched, when the storage transaction fails.
|
|
80
|
+
*/
|
|
81
|
+
commit(staging: Staging, at: {
|
|
82
|
+
cursor: number;
|
|
83
|
+
schema: string;
|
|
84
|
+
}, epoch: number): Promise<boolean>;
|
|
85
|
+
/** records that a sync found the held rows current, with no change to them */
|
|
86
|
+
touch(name: string, epoch: number): Promise<boolean>;
|
|
87
|
+
/** clears one dataset's rows and meta, and nothing else */
|
|
88
|
+
drop(name: string): Promise<void>;
|
|
89
|
+
/** clears every dataset, for a wipe of the replica */
|
|
90
|
+
clear(): Promise<void>;
|
|
91
|
+
}
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
import { applyChanges } from '../protocol/dataset.js';
|
|
2
|
+
import { compareText, TableIndex } from '../protocol/plan.js';
|
|
3
|
+
import { digestAdd, EMPTY_DIGEST } from '../protocol/query.js';
|
|
4
|
+
const encoder = new TextEncoder();
|
|
5
|
+
/** bytes of a row's JSON, the unit `maxBytes` and the storage budget count in */
|
|
6
|
+
export const rowBytes = (row) => encoder.encode(JSON.stringify(row)).length;
|
|
7
|
+
/**
|
|
8
|
+
* One dataset's state while a response or a run of seed pages is applied, before anything is
|
|
9
|
+
* written. A staged seed replaces the held rows; a staged patch changes them. Nothing here
|
|
10
|
+
* touches storage, so a failed or dropped apply leaves the store as it was.
|
|
11
|
+
*/
|
|
12
|
+
export class Staging {
|
|
13
|
+
name;
|
|
14
|
+
/** whether the commit replaces every held row */
|
|
15
|
+
clear;
|
|
16
|
+
rows;
|
|
17
|
+
hashes;
|
|
18
|
+
digest;
|
|
19
|
+
bytes;
|
|
20
|
+
#deps;
|
|
21
|
+
#puts = new Set();
|
|
22
|
+
#deletes = new Set();
|
|
23
|
+
constructor(deps, name, base) {
|
|
24
|
+
this.#deps = deps;
|
|
25
|
+
this.name = name;
|
|
26
|
+
this.clear = base === undefined;
|
|
27
|
+
this.rows = base?.rows ?? new Map();
|
|
28
|
+
this.hashes = base?.hashes ?? new Map();
|
|
29
|
+
this.digest = base?.meta.digest ?? EMPTY_DIGEST;
|
|
30
|
+
this.bytes = base?.meta.bytes ?? 0;
|
|
31
|
+
}
|
|
32
|
+
/** applies a response; returns whether the result equals the response's proof, null without one */
|
|
33
|
+
async add(response) {
|
|
34
|
+
const fresh = response.reset && response.after === undefined;
|
|
35
|
+
if (fresh) {
|
|
36
|
+
this.clear = true;
|
|
37
|
+
this.bytes = 0;
|
|
38
|
+
this.#puts.clear();
|
|
39
|
+
this.#deletes.clear();
|
|
40
|
+
}
|
|
41
|
+
// bytes follow the rows in the order the response applies them, deletes first
|
|
42
|
+
const overlay = new Map();
|
|
43
|
+
const current = (key) => overlay.has(key) ? overlay.get(key) : fresh ? undefined : this.rows.get(key);
|
|
44
|
+
for (const { key } of response.deletes) {
|
|
45
|
+
const old = current(key);
|
|
46
|
+
if (old !== undefined)
|
|
47
|
+
this.bytes -= rowBytes(old);
|
|
48
|
+
overlay.set(key, undefined);
|
|
49
|
+
}
|
|
50
|
+
for (const { key, row } of response.puts) {
|
|
51
|
+
const old = current(key);
|
|
52
|
+
if (old !== undefined)
|
|
53
|
+
this.bytes -= rowBytes(old);
|
|
54
|
+
this.bytes += rowBytes(row);
|
|
55
|
+
overlay.set(key, row);
|
|
56
|
+
}
|
|
57
|
+
const options = {
|
|
58
|
+
hashes: this.hashes,
|
|
59
|
+
...(this.#deps.hash ? { hash: this.#deps.hash } : {})
|
|
60
|
+
};
|
|
61
|
+
const applied = await applyChanges(this.rows, response, this.digest, options);
|
|
62
|
+
this.rows = applied.rows;
|
|
63
|
+
this.hashes = applied.hashes;
|
|
64
|
+
this.digest = applied.digest;
|
|
65
|
+
for (const { key } of response.deletes) {
|
|
66
|
+
this.#deletes.add(key);
|
|
67
|
+
this.#puts.delete(key);
|
|
68
|
+
}
|
|
69
|
+
for (const { key } of response.puts) {
|
|
70
|
+
this.#puts.add(key);
|
|
71
|
+
this.#deletes.delete(key);
|
|
72
|
+
}
|
|
73
|
+
return applied.matches;
|
|
74
|
+
}
|
|
75
|
+
/** the rows a commit writes: all of them for a replacement, the touched ones for a patch */
|
|
76
|
+
writes() {
|
|
77
|
+
const keys = this.clear ? [...this.rows.keys()] : [...this.#puts];
|
|
78
|
+
return {
|
|
79
|
+
puts: keys.map((key) => ({
|
|
80
|
+
name: this.name,
|
|
81
|
+
key,
|
|
82
|
+
row: this.rows.get(key),
|
|
83
|
+
hash: this.hashes.get(key)
|
|
84
|
+
})),
|
|
85
|
+
deletes: this.clear ? [] : [...this.#deletes]
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* The datasets a device holds. Rows live once in memory after a verified load and once in
|
|
91
|
+
* storage, and a response is applied to a {@link Staging} first and committed in one storage
|
|
92
|
+
* transaction. Reads and writes of the held set go through one queue, so a commit and a wipe
|
|
93
|
+
* never interleave.
|
|
94
|
+
*/
|
|
95
|
+
export class DatasetStore {
|
|
96
|
+
#deps;
|
|
97
|
+
#held = new Map();
|
|
98
|
+
#loaded = new Map();
|
|
99
|
+
#metas;
|
|
100
|
+
#queue = Promise.resolve();
|
|
101
|
+
#sorted = new Map();
|
|
102
|
+
/** the key-ordered tables and their indexes a plan reads; dropped on a commit, a drop or a wipe */
|
|
103
|
+
#tables = new Map();
|
|
104
|
+
/** datasets found damaged on a load and cleared, since the worker started */
|
|
105
|
+
evicted = 0;
|
|
106
|
+
constructor(deps) {
|
|
107
|
+
this.#deps = deps;
|
|
108
|
+
}
|
|
109
|
+
#serial(fn) {
|
|
110
|
+
const run = this.#queue.then(fn, fn);
|
|
111
|
+
this.#queue = run.catch(() => undefined);
|
|
112
|
+
return run;
|
|
113
|
+
}
|
|
114
|
+
#metaMap() {
|
|
115
|
+
this.#metas ??= this.#deps.storage
|
|
116
|
+
.getDatasets()
|
|
117
|
+
.then((all) => new Map(all.map((meta) => [meta.name, meta])));
|
|
118
|
+
return this.#metas;
|
|
119
|
+
}
|
|
120
|
+
/** the meta of every held dataset */
|
|
121
|
+
async list() {
|
|
122
|
+
return [...(await this.#metaMap()).values()].sort((a, b) => (a.name < b.name ? -1 : 1));
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Loads one dataset into memory and checks it: the stored row count and the sum of the stored row
|
|
126
|
+
* hashes must equal the meta. A dataset that fails, which is what a partly evicted store looks
|
|
127
|
+
* like, is cleared and reads as absent. No row is hashed.
|
|
128
|
+
*/
|
|
129
|
+
get(name) {
|
|
130
|
+
let held = this.#held.get(name);
|
|
131
|
+
if (!held) {
|
|
132
|
+
held = this.#load(name);
|
|
133
|
+
this.#held.set(name, held);
|
|
134
|
+
}
|
|
135
|
+
return held;
|
|
136
|
+
}
|
|
137
|
+
async #load(name) {
|
|
138
|
+
const meta = (await this.#metaMap()).get(name);
|
|
139
|
+
if (!meta)
|
|
140
|
+
return undefined;
|
|
141
|
+
const records = await this.#deps.storage.readDataset(name);
|
|
142
|
+
let sum = EMPTY_DIGEST;
|
|
143
|
+
let sound = records.length === meta.count;
|
|
144
|
+
try {
|
|
145
|
+
for (const record of records)
|
|
146
|
+
sum = digestAdd(sum, record.hash);
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
sound = false;
|
|
150
|
+
}
|
|
151
|
+
if (!sound || sum !== meta.digest) {
|
|
152
|
+
this.evicted += 1;
|
|
153
|
+
await this.drop(name);
|
|
154
|
+
return undefined;
|
|
155
|
+
}
|
|
156
|
+
const held = {
|
|
157
|
+
meta,
|
|
158
|
+
rows: new Map(records.map((r) => [r.key, r.row])),
|
|
159
|
+
hashes: new Map(records.map((r) => [r.key, r.hash]))
|
|
160
|
+
};
|
|
161
|
+
this.#loaded.set(name, held);
|
|
162
|
+
return held;
|
|
163
|
+
}
|
|
164
|
+
/** the held rows in ascending key order, or undefined when `name` is not loaded and held */
|
|
165
|
+
async rows(name) {
|
|
166
|
+
const held = await this.get(name);
|
|
167
|
+
if (!held)
|
|
168
|
+
return undefined;
|
|
169
|
+
let sorted = this.#sorted.get(name);
|
|
170
|
+
if (!sorted) {
|
|
171
|
+
sorted = [...held.rows.keys()].sort().map((key) => held.rows.get(key));
|
|
172
|
+
this.#sorted.set(name, sorted);
|
|
173
|
+
}
|
|
174
|
+
return sorted;
|
|
175
|
+
}
|
|
176
|
+
/** the base table and each joined table a plan reads, or undefined when any of them is not held */
|
|
177
|
+
async tables(base, joins) {
|
|
178
|
+
const baseTable = await this.#table(base);
|
|
179
|
+
if (!baseTable)
|
|
180
|
+
return undefined;
|
|
181
|
+
const joined = new Map();
|
|
182
|
+
for (const name of joins) {
|
|
183
|
+
const table = await this.#table(name);
|
|
184
|
+
if (!table)
|
|
185
|
+
return undefined;
|
|
186
|
+
joined.set(name, table);
|
|
187
|
+
}
|
|
188
|
+
return { base: baseTable, joined: (name) => joined.get(name) };
|
|
189
|
+
}
|
|
190
|
+
async #table(name) {
|
|
191
|
+
const held = await this.get(name);
|
|
192
|
+
if (!held)
|
|
193
|
+
return undefined;
|
|
194
|
+
let table = this.#tables.get(name);
|
|
195
|
+
if (!table) {
|
|
196
|
+
const keys = [...held.rows.keys()].sort(compareText);
|
|
197
|
+
table = { keys, rows: held.rows, index: new TableIndex(keys, held.rows) };
|
|
198
|
+
this.#tables.set(name, table);
|
|
199
|
+
}
|
|
200
|
+
return table;
|
|
201
|
+
}
|
|
202
|
+
/** whether `name` is held and in memory */
|
|
203
|
+
loaded(name) {
|
|
204
|
+
return this.#loaded.has(name);
|
|
205
|
+
}
|
|
206
|
+
/** a held row, looked up in memory; undefined unless {@link DatasetStore.get} loaded the dataset */
|
|
207
|
+
row(name, key) {
|
|
208
|
+
return this.#loaded.get(name)?.rows.get(key);
|
|
209
|
+
}
|
|
210
|
+
/** a staging area over the held rows of `name`, or an empty one for a seed */
|
|
211
|
+
stage(name, base) {
|
|
212
|
+
return new Staging(this.#deps, name, base);
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Writes a staged apply. `cursor` and `schema` are what the response brought the rows to.
|
|
216
|
+
* Returns false, writing nothing, when the replica was wiped since `epoch` was read. Rejects,
|
|
217
|
+
* with the held rows and the memory copy untouched, when the storage transaction fails.
|
|
218
|
+
*/
|
|
219
|
+
commit(staging, at, epoch) {
|
|
220
|
+
return this.#serial(async () => {
|
|
221
|
+
if (this.#deps.epoch() !== epoch)
|
|
222
|
+
return false;
|
|
223
|
+
const before = this.#loaded.get(staging.name);
|
|
224
|
+
const meta = {
|
|
225
|
+
name: staging.name,
|
|
226
|
+
schema: at.schema,
|
|
227
|
+
cursor: at.cursor,
|
|
228
|
+
count: staging.rows.size,
|
|
229
|
+
digest: staging.digest,
|
|
230
|
+
bytes: staging.bytes,
|
|
231
|
+
appliedAt: this.#deps.now()
|
|
232
|
+
};
|
|
233
|
+
const { puts, deletes } = staging.writes();
|
|
234
|
+
await this.#deps.storage.commitDataset({
|
|
235
|
+
name: staging.name,
|
|
236
|
+
meta,
|
|
237
|
+
clear: staging.clear,
|
|
238
|
+
puts,
|
|
239
|
+
deletes
|
|
240
|
+
});
|
|
241
|
+
const held = { meta, rows: staging.rows, hashes: staging.hashes };
|
|
242
|
+
this.#tables.delete(staging.name);
|
|
243
|
+
await this.#set(staging.name, held);
|
|
244
|
+
await this.#deps.onChange?.(staging.name, ...changedKeys(before, held));
|
|
245
|
+
return true;
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
/** records that a sync found the held rows current, with no change to them */
|
|
249
|
+
touch(name, epoch) {
|
|
250
|
+
return this.#serial(async () => {
|
|
251
|
+
const held = this.#loaded.get(name);
|
|
252
|
+
if (!held || this.#deps.epoch() !== epoch)
|
|
253
|
+
return false;
|
|
254
|
+
const meta = { ...held.meta, appliedAt: this.#deps.now() };
|
|
255
|
+
await this.#deps.storage.commitDataset({
|
|
256
|
+
name,
|
|
257
|
+
meta,
|
|
258
|
+
clear: false,
|
|
259
|
+
puts: [],
|
|
260
|
+
deletes: []
|
|
261
|
+
});
|
|
262
|
+
await this.#set(name, { ...held, meta });
|
|
263
|
+
return true;
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
/** clears one dataset's rows and meta, and nothing else */
|
|
267
|
+
drop(name) {
|
|
268
|
+
return this.#serial(async () => {
|
|
269
|
+
const before = this.#loaded.get(name);
|
|
270
|
+
await this.#deps.storage.commitDataset({
|
|
271
|
+
name,
|
|
272
|
+
meta: null,
|
|
273
|
+
clear: true,
|
|
274
|
+
puts: [],
|
|
275
|
+
deletes: []
|
|
276
|
+
});
|
|
277
|
+
this.#tables.delete(name);
|
|
278
|
+
await this.#set(name, undefined);
|
|
279
|
+
if (before)
|
|
280
|
+
await this.#deps.onChange?.(name, [], [...before.rows.keys()]);
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
/** clears every dataset, for a wipe of the replica */
|
|
284
|
+
clear() {
|
|
285
|
+
return this.#serial(async () => {
|
|
286
|
+
await this.#deps.storage.clearDatasets();
|
|
287
|
+
this.#held.clear();
|
|
288
|
+
this.#loaded.clear();
|
|
289
|
+
this.#sorted.clear();
|
|
290
|
+
this.#tables.clear();
|
|
291
|
+
this.#metas = Promise.resolve(new Map());
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
async #set(name, held) {
|
|
295
|
+
const metas = await this.#metaMap();
|
|
296
|
+
this.#sorted.delete(name);
|
|
297
|
+
if (held) {
|
|
298
|
+
metas.set(name, held.meta);
|
|
299
|
+
this.#loaded.set(name, held);
|
|
300
|
+
this.#held.set(name, Promise.resolve(held));
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
metas.delete(name);
|
|
304
|
+
this.#loaded.delete(name);
|
|
305
|
+
this.#held.delete(name);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
/** the keys a commit put (new or different rows) and removed, by comparing row hashes */
|
|
310
|
+
function changedKeys(before, after) {
|
|
311
|
+
const puts = [];
|
|
312
|
+
for (const [key, hash] of after.hashes) {
|
|
313
|
+
if (before?.hashes.get(key) !== hash)
|
|
314
|
+
puts.push(key);
|
|
315
|
+
}
|
|
316
|
+
const deletes = [...(before?.hashes.keys() ?? [])].filter((key) => !after.hashes.has(key));
|
|
317
|
+
return [puts, deletes];
|
|
318
|
+
}
|
package/dist/sw/graph/index.d.ts
CHANGED
|
@@ -11,13 +11,17 @@ export type Invalidated = {
|
|
|
11
11
|
projections: string[];
|
|
12
12
|
artifacts: string[];
|
|
13
13
|
};
|
|
14
|
-
/**
|
|
14
|
+
/**
|
|
15
|
+
* a page with every reference resolved, or why it cannot be built from what is held; `datasets`
|
|
16
|
+
* names the datasets the page references that the device does not hold, which a seed would fix
|
|
17
|
+
*/
|
|
15
18
|
export type Materialized = {
|
|
16
19
|
ok: true;
|
|
17
20
|
artifact: Artifact;
|
|
18
21
|
} | {
|
|
19
22
|
ok: false;
|
|
20
23
|
reason: string;
|
|
24
|
+
datasets?: string[];
|
|
21
25
|
};
|
|
22
26
|
/** what the graph reads from the replica around it */
|
|
23
27
|
export type GraphDeps = {
|
|
@@ -26,6 +30,22 @@ export type GraphDeps = {
|
|
|
26
30
|
tagVersion(tag: string): number;
|
|
27
31
|
/** every stored artifact, read once to build the index on first use */
|
|
28
32
|
artifacts(): Promise<StoredArtifact[]>;
|
|
33
|
+
/** the held row `key` of `dataset`; undefined when it is not held or its dataset is not loaded */
|
|
34
|
+
row?(dataset: string, key: string): unknown;
|
|
35
|
+
/** whether `dataset` is held and loaded */
|
|
36
|
+
held?(dataset: string): boolean;
|
|
37
|
+
/** loads these datasets into memory before a page that references their rows is resolved */
|
|
38
|
+
loadDatasets?(names: string[]): Promise<void>;
|
|
39
|
+
};
|
|
40
|
+
/** what a change to dataset rows did to the graph */
|
|
41
|
+
export type RowEffect = {
|
|
42
|
+
/** projections and artifacts that reference a changed row; they resolve it again at render, so nothing is fetched */
|
|
43
|
+
touched: {
|
|
44
|
+
projections: string[];
|
|
45
|
+
artifacts: string[];
|
|
46
|
+
};
|
|
47
|
+
/** what a deleted row left unresolvable, dropped from the graph; the replica drops the artifacts */
|
|
48
|
+
dropped: Invalidated;
|
|
29
49
|
};
|
|
30
50
|
/** whether an artifact references anything in the graph */
|
|
31
51
|
export declare function usesGraph(artifact: Artifact): boolean;
|
|
@@ -60,8 +80,18 @@ export declare class Graph {
|
|
|
60
80
|
* was. Artifacts that reference anything dropped are reported, not removed: the replica owns them.
|
|
61
81
|
*/
|
|
62
82
|
invalidate(changed: Readonly<Record<string, number>>): Promise<Invalidated>;
|
|
83
|
+
/**
|
|
84
|
+
* Applies a change to the rows of dataset `dataset`. A page or projection that references a
|
|
85
|
+
* row holds its id, so a put needs no repair: the row resolves again at the next render, and
|
|
86
|
+
* only the pages that reference it are reported as touched. A deleted row leaves the pages that
|
|
87
|
+
* reference it unresolvable, so they and the projections above them are dropped. Nothing
|
|
88
|
+
* else is looked at.
|
|
89
|
+
*/
|
|
90
|
+
rowsChanged(dataset: string, puts: readonly string[], deletes: readonly string[]): Promise<RowEffect>;
|
|
63
91
|
/** forgets everything in memory and in storage */
|
|
64
92
|
clear(): Promise<void>;
|
|
93
|
+
/** the datasets whose rows the artifact references, directly or through projections it reaches */
|
|
94
|
+
datasetsOf(artifact: Artifact): string[];
|
|
65
95
|
/** the artifact with every reference replaced by what the graph holds */
|
|
66
96
|
materialize(artifact: Artifact): Promise<Materialized>;
|
|
67
97
|
}
|
package/dist/sw/graph/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { collectRefs, resolveRefs } from '../../protocol/graph.js';
|
|
1
|
+
import { collectRefs, resolveRefs, splitRowId } from '../../protocol/graph.js';
|
|
2
2
|
import { DependencyIndex } from './invalidate.js';
|
|
3
3
|
import { projectionStore } from './projections.js';
|
|
4
4
|
import { recordStore } from './records.js';
|
|
@@ -12,10 +12,11 @@ export function usesGraph(artifact) {
|
|
|
12
12
|
artifact.projections !== undefined ||
|
|
13
13
|
hasRefs(collectRefs(artifact.body)));
|
|
14
14
|
}
|
|
15
|
-
const hasRefs = (refs) => refs.records.length + refs.projections.length > 0;
|
|
15
|
+
const hasRefs = (refs) => refs.records.length + refs.projections.length + (refs.rows?.length ?? 0) > 0;
|
|
16
16
|
function childProjections(entry) {
|
|
17
17
|
return collectRefs(entry.data).projections;
|
|
18
18
|
}
|
|
19
|
+
const rowsOf = (entry) => collectRefs(entry.data).rows ?? [];
|
|
19
20
|
/**
|
|
20
21
|
* Records and projections stored once, the index over them, and the rules that tie them to
|
|
21
22
|
* artifacts. Nothing is read from storage until {@link Graph.ensure}: a worker that wakes for a
|
|
@@ -49,7 +50,7 @@ export class Graph {
|
|
|
49
50
|
async #load() {
|
|
50
51
|
await Promise.all([this.records.load(), this.projections.load()]);
|
|
51
52
|
for (const entry of this.projections.all()) {
|
|
52
|
-
this.index.addProjection(entry.id, entry.deps, childProjections(entry));
|
|
53
|
+
this.index.addProjection(entry.id, entry.deps, childProjections(entry), rowsOf(entry));
|
|
53
54
|
}
|
|
54
55
|
for (const { artifact } of await this.#deps.artifacts()) {
|
|
55
56
|
const uses = collectRefs(artifact.body);
|
|
@@ -77,7 +78,7 @@ export class Graph {
|
|
|
77
78
|
const added = await this.projections.put(artifact.projections ?? [], persist);
|
|
78
79
|
for (const id of added) {
|
|
79
80
|
const entry = this.projections.get(id);
|
|
80
|
-
this.index.addProjection(id, entry.deps, childProjections(entry));
|
|
81
|
+
this.index.addProjection(id, entry.deps, childProjections(entry), rowsOf(entry));
|
|
81
82
|
}
|
|
82
83
|
const uses = collectRefs(artifact.body);
|
|
83
84
|
if (hasRefs(uses))
|
|
@@ -118,6 +119,24 @@ export class Graph {
|
|
|
118
119
|
this.index.removeProjections(projections);
|
|
119
120
|
return { records: records.sort(), projections, artifacts };
|
|
120
121
|
}
|
|
122
|
+
/**
|
|
123
|
+
* Applies a change to the rows of dataset `dataset`. A page or projection that references a
|
|
124
|
+
* row holds its id, so a put needs no repair: the row resolves again at the next render, and
|
|
125
|
+
* only the pages that reference it are reported as touched. A deleted row leaves the pages that
|
|
126
|
+
* reference it unresolvable, so they and the projections above them are dropped. Nothing
|
|
127
|
+
* else is looked at.
|
|
128
|
+
*/
|
|
129
|
+
async rowsChanged(dataset, puts, deletes) {
|
|
130
|
+
const id = (key) => `${dataset}/${key}`;
|
|
131
|
+
const touched = this.index.rowsReaching([...puts, ...deletes].map(id));
|
|
132
|
+
const gone = this.index.rowsReaching(deletes.map(id));
|
|
133
|
+
await this.projections.drop(gone.projections);
|
|
134
|
+
this.index.removeProjections(gone.projections);
|
|
135
|
+
return {
|
|
136
|
+
touched,
|
|
137
|
+
dropped: { records: [], projections: gone.projections, artifacts: gone.artifacts }
|
|
138
|
+
};
|
|
139
|
+
}
|
|
121
140
|
/** forgets everything in memory and in storage */
|
|
122
141
|
async clear() {
|
|
123
142
|
this.records.forget();
|
|
@@ -125,15 +144,48 @@ export class Graph {
|
|
|
125
144
|
this.index.clear();
|
|
126
145
|
await this.#deps.storage.clearGraph();
|
|
127
146
|
}
|
|
147
|
+
/** the datasets whose rows the artifact references, directly or through projections it reaches */
|
|
148
|
+
datasetsOf(artifact) {
|
|
149
|
+
const names = new Set();
|
|
150
|
+
const seen = new Set();
|
|
151
|
+
const queue = [artifact.body];
|
|
152
|
+
for (let node = queue.pop(); node !== undefined; node = queue.pop()) {
|
|
153
|
+
const refs = collectRefs(node);
|
|
154
|
+
for (const id of refs.rows ?? [])
|
|
155
|
+
names.add(splitRowId(id)?.dataset ?? id);
|
|
156
|
+
for (const id of refs.projections) {
|
|
157
|
+
if (seen.has(id))
|
|
158
|
+
continue;
|
|
159
|
+
seen.add(id);
|
|
160
|
+
const entry = this.projections.get(id);
|
|
161
|
+
if (entry)
|
|
162
|
+
queue.push(entry.data);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return [...names];
|
|
166
|
+
}
|
|
128
167
|
/** the artifact with every reference replaced by what the graph holds */
|
|
129
168
|
async materialize(artifact) {
|
|
130
169
|
if (artifact.strategy === 'static' || !hasRefs(collectRefs(artifact.body))) {
|
|
131
170
|
return { ok: true, artifact };
|
|
132
171
|
}
|
|
133
172
|
await this.ensure();
|
|
173
|
+
const datasets = this.datasetsOf(artifact);
|
|
174
|
+
if (datasets.length > 0) {
|
|
175
|
+
await this.#deps.loadDatasets?.(datasets);
|
|
176
|
+
const missing = datasets.filter((name) => !this.#deps.held?.(name));
|
|
177
|
+
if (missing.length > 0) {
|
|
178
|
+
return {
|
|
179
|
+
ok: false,
|
|
180
|
+
reason: `dataset ${missing[0]} is not held`,
|
|
181
|
+
datasets: missing
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
}
|
|
134
185
|
const lookup = {
|
|
135
186
|
record: (id) => this.records.get(id),
|
|
136
|
-
projection: (id) => this.projections.get(id)
|
|
187
|
+
projection: (id) => this.projections.get(id),
|
|
188
|
+
...(this.#deps.row ? { row: this.#deps.row } : {})
|
|
137
189
|
};
|
|
138
190
|
if (artifact.strategy !== 'assembly') {
|
|
139
191
|
const data = resolveRefs(artifact.body.data, lookup);
|
|
@@ -6,14 +6,26 @@ import type { RefSet } from '../../protocol/graph.js';
|
|
|
6
6
|
*/
|
|
7
7
|
export declare class DependencyIndex {
|
|
8
8
|
#private;
|
|
9
|
-
/**
|
|
10
|
-
|
|
9
|
+
/**
|
|
10
|
+
* records that `projection` reads the tags in `deps`, references the projections in `children`
|
|
11
|
+
* and the dataset rows in `rows`
|
|
12
|
+
*/
|
|
13
|
+
addProjection(projection: string, deps: readonly string[], children: readonly string[], rows?: readonly string[]): void;
|
|
11
14
|
removeProjections(ids: readonly string[]): void;
|
|
12
|
-
/** records that `artifact` references these records and
|
|
15
|
+
/** records that `artifact` references these records, projections and dataset rows */
|
|
13
16
|
addArtifact(artifact: string, uses: RefSet): void;
|
|
14
17
|
removeArtifacts(ids: readonly string[]): void;
|
|
15
18
|
/** forgets everything */
|
|
16
19
|
clear(): void;
|
|
20
|
+
/**
|
|
21
|
+
* What a change to dataset rows reaches: the projections that reference one directly, those
|
|
22
|
+
* and every projection above them, and the artifacts that use a reached projection or one of
|
|
23
|
+
* the rows. Row ids are `<dataset>/<key>`.
|
|
24
|
+
*/
|
|
25
|
+
rowsReaching(rows: Iterable<string>): {
|
|
26
|
+
projections: string[];
|
|
27
|
+
artifacts: string[];
|
|
28
|
+
};
|
|
17
29
|
/** the projections that read any of `tags` directly */
|
|
18
30
|
projectionsReading(tags: Iterable<string>): string[];
|
|
19
31
|
/** `projections` and every projection that references one of them, directly or not */
|