@drupflare/spool 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +263 -41
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/entry.js +33 -1
  6. package/dist/client/register.d.ts +23 -1
  7. package/dist/client/register.js +53 -7
  8. package/dist/protocol/config.d.ts +16 -1
  9. package/dist/protocol/config.js +3 -1
  10. package/dist/protocol/dataset.d.ts +183 -0
  11. package/dist/protocol/dataset.js +206 -0
  12. package/dist/protocol/envelope.d.ts +1 -0
  13. package/dist/protocol/envelope.js +2 -1
  14. package/dist/protocol/errors.d.ts +1 -1
  15. package/dist/protocol/graph.d.ts +23 -4
  16. package/dist/protocol/graph.js +34 -3
  17. package/dist/protocol/index.d.ts +3 -0
  18. package/dist/protocol/index.js +3 -0
  19. package/dist/protocol/plan.d.ts +100 -0
  20. package/dist/protocol/plan.js +661 -0
  21. package/dist/protocol/provision.d.ts +46 -0
  22. package/dist/protocol/provision.js +38 -0
  23. package/dist/protocol/query.d.ts +66 -5
  24. package/dist/protocol/query.js +103 -4
  25. package/dist/protocol/route.d.ts +6 -2
  26. package/dist/protocol/route.js +31 -3
  27. package/dist/protocol/write.d.ts +146 -0
  28. package/dist/protocol/write.js +101 -0
  29. package/dist/report/dataset.d.ts +48 -0
  30. package/dist/report/dataset.js +93 -0
  31. package/dist/report/derive.d.ts +285 -0
  32. package/dist/report/derive.js +676 -1
  33. package/dist/report/economics.d.ts +11 -1
  34. package/dist/report/economics.js +19 -1
  35. package/dist/report/index.d.ts +2 -0
  36. package/dist/report/index.js +2 -0
  37. package/dist/report/inputs.d.ts +97 -0
  38. package/dist/report/inputs.js +133 -0
  39. package/dist/report/institution.d.ts +188 -1
  40. package/dist/report/institution.js +146 -0
  41. package/dist/report/loops.d.ts +73 -0
  42. package/dist/report/loops.js +101 -0
  43. package/dist/report/measurement.d.ts +1 -1
  44. package/dist/report/measurement.js +3 -1
  45. package/dist/report/model.d.ts +2 -0
  46. package/dist/report/model.js +4 -0
  47. package/dist/report/thirdparty.d.ts +57 -0
  48. package/dist/report/thirdparty.js +111 -0
  49. package/dist/report/trim.d.ts +56 -0
  50. package/dist/report/trim.js +83 -1
  51. package/dist/server/dataset.d.ts +48 -0
  52. package/dist/server/dataset.js +308 -0
  53. package/dist/server/embedded.d.ts +3 -3
  54. package/dist/server/embedded.js +3 -3
  55. package/dist/server/index.d.ts +2 -0
  56. package/dist/server/index.js +2 -0
  57. package/dist/server/spool.d.ts +20 -0
  58. package/dist/server/spool.js +126 -18
  59. package/dist/server/write.d.ts +27 -0
  60. package/dist/server/write.js +127 -0
  61. package/dist/sw/analytics.d.ts +6 -0
  62. package/dist/sw/analytics.js +14 -0
  63. package/dist/sw/dataset-sync.d.ts +76 -0
  64. package/dist/sw/dataset-sync.js +149 -0
  65. package/dist/sw/dataset.d.ts +91 -0
  66. package/dist/sw/dataset.js +318 -0
  67. package/dist/sw/graph/index.d.ts +31 -1
  68. package/dist/sw/graph/index.js +57 -5
  69. package/dist/sw/graph/invalidate.d.ts +15 -3
  70. package/dist/sw/graph/invalidate.js +42 -4
  71. package/dist/sw/idb.js +78 -9
  72. package/dist/sw/partition.d.ts +6 -1
  73. package/dist/sw/partition.js +21 -2
  74. package/dist/sw/principal-dataset.d.ts +64 -0
  75. package/dist/sw/principal-dataset.js +254 -0
  76. package/dist/sw/render.d.ts +7 -2
  77. package/dist/sw/render.js +19 -5
  78. package/dist/sw/replica.d.ts +12 -3
  79. package/dist/sw/replica.js +40 -5
  80. package/dist/sw/runtime.d.ts +43 -2
  81. package/dist/sw/runtime.js +340 -24
  82. package/dist/sw/seal.d.ts +13 -0
  83. package/dist/sw/seal.js +13 -4
  84. package/dist/sw/static.d.ts +18 -0
  85. package/dist/sw/static.js +84 -1
  86. package/dist/sw/storage.d.ts +51 -1
  87. package/dist/sw/storage.js +33 -1
  88. package/dist/sw/write.d.ts +107 -0
  89. package/dist/sw/write.js +442 -0
  90. package/package.json +3 -2
@@ -0,0 +1,91 @@
1
+ import { type DatasetResponse, type DatasetRow } from '../protocol/dataset.js';
2
+ import { type PlanTables } from '../protocol/plan.js';
3
+ import type { DatasetMeta, DatasetRowRecord, ReplicaStorage } from './storage.js';
4
+ /** bytes of a row's JSON, the unit `maxBytes` and the storage budget count in */
5
+ export declare const rowBytes: (row: DatasetRow) => number;
6
+ /** a dataset in memory, as a verified load or a commit left it */
7
+ export type Held = {
8
+ meta: DatasetMeta;
9
+ rows: Map<string, DatasetRow>;
10
+ /** the hash of every row, so a patch never hashes a row it replaces or deletes */
11
+ hashes: Map<string, string>;
12
+ };
13
+ /** what the store needs from the replica around it */
14
+ export type DatasetStoreDeps = {
15
+ storage: ReplicaStorage;
16
+ now(): number;
17
+ /** the replica's wipe count; a commit that began under another one is dropped */
18
+ epoch(): number;
19
+ /** the row hash; {@link rowHash}. A test counts calls through it */
20
+ hash?: (key: string, row: DatasetRow) => Promise<string>;
21
+ /** told after every commit which row keys it put and which it removed */
22
+ onChange?(name: string, puts: string[], deletes: string[]): Promise<void> | void;
23
+ };
24
+ /**
25
+ * One dataset's state while a response or a run of seed pages is applied, before anything is
26
+ * written. A staged seed replaces the held rows; a staged patch changes them. Nothing here
27
+ * touches storage, so a failed or dropped apply leaves the store as it was.
28
+ */
29
+ export declare class Staging {
30
+ #private;
31
+ readonly name: string;
32
+ /** whether the commit replaces every held row */
33
+ clear: boolean;
34
+ rows: Map<string, DatasetRow>;
35
+ hashes: Map<string, string>;
36
+ digest: string;
37
+ bytes: number;
38
+ constructor(deps: DatasetStoreDeps, name: string, base?: Held);
39
+ /** applies a response; returns whether the result equals the response's proof, null without one */
40
+ add(response: DatasetResponse): Promise<boolean | null>;
41
+ /** the rows a commit writes: all of them for a replacement, the touched ones for a patch */
42
+ writes(): {
43
+ puts: DatasetRowRecord[];
44
+ deletes: string[];
45
+ };
46
+ }
47
+ /**
48
+ * The datasets a device holds. Rows live once in memory after a verified load and once in
49
+ * storage, and a response is applied to a {@link Staging} first and committed in one storage
50
+ * transaction. Reads and writes of the held set go through one queue, so a commit and a wipe
51
+ * never interleave.
52
+ */
53
+ export declare class DatasetStore {
54
+ #private;
55
+ /** datasets found damaged on a load and cleared, since the worker started */
56
+ evicted: number;
57
+ constructor(deps: DatasetStoreDeps);
58
+ /** the meta of every held dataset */
59
+ list(): Promise<DatasetMeta[]>;
60
+ /**
61
+ * Loads one dataset into memory and checks it: the stored row count and the sum of the stored row
62
+ * hashes must equal the meta. A dataset that fails, which is what a partly evicted store looks
63
+ * like, is cleared and reads as absent. No row is hashed.
64
+ */
65
+ get(name: string): Promise<Held | undefined>;
66
+ /** the held rows in ascending key order, or undefined when `name` is not loaded and held */
67
+ rows(name: string): Promise<DatasetRow[] | undefined>;
68
+ /** the base table and each joined table a plan reads, or undefined when any of them is not held */
69
+ tables(base: string, joins: readonly string[]): Promise<PlanTables | undefined>;
70
+ /** whether `name` is held and in memory */
71
+ loaded(name: string): boolean;
72
+ /** a held row, looked up in memory; undefined unless {@link DatasetStore.get} loaded the dataset */
73
+ row(name: string, key: string): DatasetRow | undefined;
74
+ /** a staging area over the held rows of `name`, or an empty one for a seed */
75
+ stage(name: string, base?: Held): Staging;
76
+ /**
77
+ * Writes a staged apply. `cursor` and `schema` are what the response brought the rows to.
78
+ * Returns false, writing nothing, when the replica was wiped since `epoch` was read. Rejects,
79
+ * with the held rows and the memory copy untouched, when the storage transaction fails.
80
+ */
81
+ commit(staging: Staging, at: {
82
+ cursor: number;
83
+ schema: string;
84
+ }, epoch: number): Promise<boolean>;
85
+ /** records that a sync found the held rows current, with no change to them */
86
+ touch(name: string, epoch: number): Promise<boolean>;
87
+ /** clears one dataset's rows and meta, and nothing else */
88
+ drop(name: string): Promise<void>;
89
+ /** clears every dataset, for a wipe of the replica */
90
+ clear(): Promise<void>;
91
+ }
@@ -0,0 +1,318 @@
1
+ import { applyChanges } from '../protocol/dataset.js';
2
+ import { compareText, TableIndex } from '../protocol/plan.js';
3
+ import { digestAdd, EMPTY_DIGEST } from '../protocol/query.js';
4
+ const encoder = new TextEncoder();
5
+ /** bytes of a row's JSON, the unit `maxBytes` and the storage budget count in */
6
+ export const rowBytes = (row) => encoder.encode(JSON.stringify(row)).length;
7
+ /**
8
+ * One dataset's state while a response or a run of seed pages is applied, before anything is
9
+ * written. A staged seed replaces the held rows; a staged patch changes them. Nothing here
10
+ * touches storage, so a failed or dropped apply leaves the store as it was.
11
+ */
12
+ export class Staging {
13
+ name;
14
+ /** whether the commit replaces every held row */
15
+ clear;
16
+ rows;
17
+ hashes;
18
+ digest;
19
+ bytes;
20
+ #deps;
21
+ #puts = new Set();
22
+ #deletes = new Set();
23
+ constructor(deps, name, base) {
24
+ this.#deps = deps;
25
+ this.name = name;
26
+ this.clear = base === undefined;
27
+ this.rows = base?.rows ?? new Map();
28
+ this.hashes = base?.hashes ?? new Map();
29
+ this.digest = base?.meta.digest ?? EMPTY_DIGEST;
30
+ this.bytes = base?.meta.bytes ?? 0;
31
+ }
32
+ /** applies a response; returns whether the result equals the response's proof, null without one */
33
+ async add(response) {
34
+ const fresh = response.reset && response.after === undefined;
35
+ if (fresh) {
36
+ this.clear = true;
37
+ this.bytes = 0;
38
+ this.#puts.clear();
39
+ this.#deletes.clear();
40
+ }
41
+ // bytes follow the rows in the order the response applies them, deletes first
42
+ const overlay = new Map();
43
+ const current = (key) => overlay.has(key) ? overlay.get(key) : fresh ? undefined : this.rows.get(key);
44
+ for (const { key } of response.deletes) {
45
+ const old = current(key);
46
+ if (old !== undefined)
47
+ this.bytes -= rowBytes(old);
48
+ overlay.set(key, undefined);
49
+ }
50
+ for (const { key, row } of response.puts) {
51
+ const old = current(key);
52
+ if (old !== undefined)
53
+ this.bytes -= rowBytes(old);
54
+ this.bytes += rowBytes(row);
55
+ overlay.set(key, row);
56
+ }
57
+ const options = {
58
+ hashes: this.hashes,
59
+ ...(this.#deps.hash ? { hash: this.#deps.hash } : {})
60
+ };
61
+ const applied = await applyChanges(this.rows, response, this.digest, options);
62
+ this.rows = applied.rows;
63
+ this.hashes = applied.hashes;
64
+ this.digest = applied.digest;
65
+ for (const { key } of response.deletes) {
66
+ this.#deletes.add(key);
67
+ this.#puts.delete(key);
68
+ }
69
+ for (const { key } of response.puts) {
70
+ this.#puts.add(key);
71
+ this.#deletes.delete(key);
72
+ }
73
+ return applied.matches;
74
+ }
75
+ /** the rows a commit writes: all of them for a replacement, the touched ones for a patch */
76
+ writes() {
77
+ const keys = this.clear ? [...this.rows.keys()] : [...this.#puts];
78
+ return {
79
+ puts: keys.map((key) => ({
80
+ name: this.name,
81
+ key,
82
+ row: this.rows.get(key),
83
+ hash: this.hashes.get(key)
84
+ })),
85
+ deletes: this.clear ? [] : [...this.#deletes]
86
+ };
87
+ }
88
+ }
89
+ /**
90
+ * The datasets a device holds. Rows live once in memory after a verified load and once in
91
+ * storage, and a response is applied to a {@link Staging} first and committed in one storage
92
+ * transaction. Reads and writes of the held set go through one queue, so a commit and a wipe
93
+ * never interleave.
94
+ */
95
+ export class DatasetStore {
96
+ #deps;
97
+ #held = new Map();
98
+ #loaded = new Map();
99
+ #metas;
100
+ #queue = Promise.resolve();
101
+ #sorted = new Map();
102
+ /** the key-ordered tables and their indexes a plan reads; dropped on a commit, a drop or a wipe */
103
+ #tables = new Map();
104
+ /** datasets found damaged on a load and cleared, since the worker started */
105
+ evicted = 0;
106
+ constructor(deps) {
107
+ this.#deps = deps;
108
+ }
109
+ #serial(fn) {
110
+ const run = this.#queue.then(fn, fn);
111
+ this.#queue = run.catch(() => undefined);
112
+ return run;
113
+ }
114
+ #metaMap() {
115
+ this.#metas ??= this.#deps.storage
116
+ .getDatasets()
117
+ .then((all) => new Map(all.map((meta) => [meta.name, meta])));
118
+ return this.#metas;
119
+ }
120
+ /** the meta of every held dataset */
121
+ async list() {
122
+ return [...(await this.#metaMap()).values()].sort((a, b) => (a.name < b.name ? -1 : 1));
123
+ }
124
+ /**
125
+ * Loads one dataset into memory and checks it: the stored row count and the sum of the stored row
126
+ * hashes must equal the meta. A dataset that fails, which is what a partly evicted store looks
127
+ * like, is cleared and reads as absent. No row is hashed.
128
+ */
129
+ get(name) {
130
+ let held = this.#held.get(name);
131
+ if (!held) {
132
+ held = this.#load(name);
133
+ this.#held.set(name, held);
134
+ }
135
+ return held;
136
+ }
137
+ async #load(name) {
138
+ const meta = (await this.#metaMap()).get(name);
139
+ if (!meta)
140
+ return undefined;
141
+ const records = await this.#deps.storage.readDataset(name);
142
+ let sum = EMPTY_DIGEST;
143
+ let sound = records.length === meta.count;
144
+ try {
145
+ for (const record of records)
146
+ sum = digestAdd(sum, record.hash);
147
+ }
148
+ catch {
149
+ sound = false;
150
+ }
151
+ if (!sound || sum !== meta.digest) {
152
+ this.evicted += 1;
153
+ await this.drop(name);
154
+ return undefined;
155
+ }
156
+ const held = {
157
+ meta,
158
+ rows: new Map(records.map((r) => [r.key, r.row])),
159
+ hashes: new Map(records.map((r) => [r.key, r.hash]))
160
+ };
161
+ this.#loaded.set(name, held);
162
+ return held;
163
+ }
164
+ /** the held rows in ascending key order, or undefined when `name` is not loaded and held */
165
+ async rows(name) {
166
+ const held = await this.get(name);
167
+ if (!held)
168
+ return undefined;
169
+ let sorted = this.#sorted.get(name);
170
+ if (!sorted) {
171
+ sorted = [...held.rows.keys()].sort().map((key) => held.rows.get(key));
172
+ this.#sorted.set(name, sorted);
173
+ }
174
+ return sorted;
175
+ }
176
+ /** the base table and each joined table a plan reads, or undefined when any of them is not held */
177
+ async tables(base, joins) {
178
+ const baseTable = await this.#table(base);
179
+ if (!baseTable)
180
+ return undefined;
181
+ const joined = new Map();
182
+ for (const name of joins) {
183
+ const table = await this.#table(name);
184
+ if (!table)
185
+ return undefined;
186
+ joined.set(name, table);
187
+ }
188
+ return { base: baseTable, joined: (name) => joined.get(name) };
189
+ }
190
+ async #table(name) {
191
+ const held = await this.get(name);
192
+ if (!held)
193
+ return undefined;
194
+ let table = this.#tables.get(name);
195
+ if (!table) {
196
+ const keys = [...held.rows.keys()].sort(compareText);
197
+ table = { keys, rows: held.rows, index: new TableIndex(keys, held.rows) };
198
+ this.#tables.set(name, table);
199
+ }
200
+ return table;
201
+ }
202
+ /** whether `name` is held and in memory */
203
+ loaded(name) {
204
+ return this.#loaded.has(name);
205
+ }
206
+ /** a held row, looked up in memory; undefined unless {@link DatasetStore.get} loaded the dataset */
207
+ row(name, key) {
208
+ return this.#loaded.get(name)?.rows.get(key);
209
+ }
210
+ /** a staging area over the held rows of `name`, or an empty one for a seed */
211
+ stage(name, base) {
212
+ return new Staging(this.#deps, name, base);
213
+ }
214
+ /**
215
+ * Writes a staged apply. `cursor` and `schema` are what the response brought the rows to.
216
+ * Returns false, writing nothing, when the replica was wiped since `epoch` was read. Rejects,
217
+ * with the held rows and the memory copy untouched, when the storage transaction fails.
218
+ */
219
+ commit(staging, at, epoch) {
220
+ return this.#serial(async () => {
221
+ if (this.#deps.epoch() !== epoch)
222
+ return false;
223
+ const before = this.#loaded.get(staging.name);
224
+ const meta = {
225
+ name: staging.name,
226
+ schema: at.schema,
227
+ cursor: at.cursor,
228
+ count: staging.rows.size,
229
+ digest: staging.digest,
230
+ bytes: staging.bytes,
231
+ appliedAt: this.#deps.now()
232
+ };
233
+ const { puts, deletes } = staging.writes();
234
+ await this.#deps.storage.commitDataset({
235
+ name: staging.name,
236
+ meta,
237
+ clear: staging.clear,
238
+ puts,
239
+ deletes
240
+ });
241
+ const held = { meta, rows: staging.rows, hashes: staging.hashes };
242
+ this.#tables.delete(staging.name);
243
+ await this.#set(staging.name, held);
244
+ await this.#deps.onChange?.(staging.name, ...changedKeys(before, held));
245
+ return true;
246
+ });
247
+ }
248
+ /** records that a sync found the held rows current, with no change to them */
249
+ touch(name, epoch) {
250
+ return this.#serial(async () => {
251
+ const held = this.#loaded.get(name);
252
+ if (!held || this.#deps.epoch() !== epoch)
253
+ return false;
254
+ const meta = { ...held.meta, appliedAt: this.#deps.now() };
255
+ await this.#deps.storage.commitDataset({
256
+ name,
257
+ meta,
258
+ clear: false,
259
+ puts: [],
260
+ deletes: []
261
+ });
262
+ await this.#set(name, { ...held, meta });
263
+ return true;
264
+ });
265
+ }
266
+ /** clears one dataset's rows and meta, and nothing else */
267
+ drop(name) {
268
+ return this.#serial(async () => {
269
+ const before = this.#loaded.get(name);
270
+ await this.#deps.storage.commitDataset({
271
+ name,
272
+ meta: null,
273
+ clear: true,
274
+ puts: [],
275
+ deletes: []
276
+ });
277
+ this.#tables.delete(name);
278
+ await this.#set(name, undefined);
279
+ if (before)
280
+ await this.#deps.onChange?.(name, [], [...before.rows.keys()]);
281
+ });
282
+ }
283
+ /** clears every dataset, for a wipe of the replica */
284
+ clear() {
285
+ return this.#serial(async () => {
286
+ await this.#deps.storage.clearDatasets();
287
+ this.#held.clear();
288
+ this.#loaded.clear();
289
+ this.#sorted.clear();
290
+ this.#tables.clear();
291
+ this.#metas = Promise.resolve(new Map());
292
+ });
293
+ }
294
+ async #set(name, held) {
295
+ const metas = await this.#metaMap();
296
+ this.#sorted.delete(name);
297
+ if (held) {
298
+ metas.set(name, held.meta);
299
+ this.#loaded.set(name, held);
300
+ this.#held.set(name, Promise.resolve(held));
301
+ }
302
+ else {
303
+ metas.delete(name);
304
+ this.#loaded.delete(name);
305
+ this.#held.delete(name);
306
+ }
307
+ }
308
+ }
309
+ /** the keys a commit put (new or different rows) and removed, by comparing row hashes */
310
+ function changedKeys(before, after) {
311
+ const puts = [];
312
+ for (const [key, hash] of after.hashes) {
313
+ if (before?.hashes.get(key) !== hash)
314
+ puts.push(key);
315
+ }
316
+ const deletes = [...(before?.hashes.keys() ?? [])].filter((key) => !after.hashes.has(key));
317
+ return [puts, deletes];
318
+ }
@@ -11,13 +11,17 @@ export type Invalidated = {
11
11
  projections: string[];
12
12
  artifacts: string[];
13
13
  };
14
- /** a page with every reference resolved, or why it cannot be built from what is held */
14
+ /**
15
+ * a page with every reference resolved, or why it cannot be built from what is held; `datasets`
16
+ * names the datasets the page references that the device does not hold, which a seed would fix
17
+ */
15
18
  export type Materialized = {
16
19
  ok: true;
17
20
  artifact: Artifact;
18
21
  } | {
19
22
  ok: false;
20
23
  reason: string;
24
+ datasets?: string[];
21
25
  };
22
26
  /** what the graph reads from the replica around it */
23
27
  export type GraphDeps = {
@@ -26,6 +30,22 @@ export type GraphDeps = {
26
30
  tagVersion(tag: string): number;
27
31
  /** every stored artifact, read once to build the index on first use */
28
32
  artifacts(): Promise<StoredArtifact[]>;
33
+ /** the held row `key` of `dataset`; undefined when it is not held or its dataset is not loaded */
34
+ row?(dataset: string, key: string): unknown;
35
+ /** whether `dataset` is held and loaded */
36
+ held?(dataset: string): boolean;
37
+ /** loads these datasets into memory before a page that references their rows is resolved */
38
+ loadDatasets?(names: string[]): Promise<void>;
39
+ };
40
+ /** what a change to dataset rows did to the graph */
41
+ export type RowEffect = {
42
+ /** projections and artifacts that reference a changed row; they resolve it again at render, so nothing is fetched */
43
+ touched: {
44
+ projections: string[];
45
+ artifacts: string[];
46
+ };
47
+ /** what a deleted row left unresolvable, dropped from the graph; the replica drops the artifacts */
48
+ dropped: Invalidated;
29
49
  };
30
50
  /** whether an artifact references anything in the graph */
31
51
  export declare function usesGraph(artifact: Artifact): boolean;
@@ -60,8 +80,18 @@ export declare class Graph {
60
80
  * was. Artifacts that reference anything dropped are reported, not removed: the replica owns them.
61
81
  */
62
82
  invalidate(changed: Readonly<Record<string, number>>): Promise<Invalidated>;
83
+ /**
84
+ * Applies a change to the rows of dataset `dataset`. A page or projection that references a
85
+ * row holds its id, so a put needs no repair: the row resolves again at the next render, and
86
+ * only the pages that reference it are reported as touched. A deleted row leaves the pages that
87
+ * reference it unresolvable, so they and the projections above them are dropped. Nothing
88
+ * else is looked at.
89
+ */
90
+ rowsChanged(dataset: string, puts: readonly string[], deletes: readonly string[]): Promise<RowEffect>;
63
91
  /** forgets everything in memory and in storage */
64
92
  clear(): Promise<void>;
93
+ /** the datasets whose rows the artifact references, directly or through projections it reaches */
94
+ datasetsOf(artifact: Artifact): string[];
65
95
  /** the artifact with every reference replaced by what the graph holds */
66
96
  materialize(artifact: Artifact): Promise<Materialized>;
67
97
  }
@@ -1,4 +1,4 @@
1
- import { collectRefs, resolveRefs } from '../../protocol/graph.js';
1
+ import { collectRefs, resolveRefs, splitRowId } from '../../protocol/graph.js';
2
2
  import { DependencyIndex } from './invalidate.js';
3
3
  import { projectionStore } from './projections.js';
4
4
  import { recordStore } from './records.js';
@@ -12,10 +12,11 @@ export function usesGraph(artifact) {
12
12
  artifact.projections !== undefined ||
13
13
  hasRefs(collectRefs(artifact.body)));
14
14
  }
15
- const hasRefs = (refs) => refs.records.length + refs.projections.length > 0;
15
+ const hasRefs = (refs) => refs.records.length + refs.projections.length + (refs.rows?.length ?? 0) > 0;
16
16
  function childProjections(entry) {
17
17
  return collectRefs(entry.data).projections;
18
18
  }
19
+ const rowsOf = (entry) => collectRefs(entry.data).rows ?? [];
19
20
  /**
20
21
  * Records and projections stored once, the index over them, and the rules that tie them to
21
22
  * artifacts. Nothing is read from storage until {@link Graph.ensure}: a worker that wakes for a
@@ -49,7 +50,7 @@ export class Graph {
49
50
  async #load() {
50
51
  await Promise.all([this.records.load(), this.projections.load()]);
51
52
  for (const entry of this.projections.all()) {
52
- this.index.addProjection(entry.id, entry.deps, childProjections(entry));
53
+ this.index.addProjection(entry.id, entry.deps, childProjections(entry), rowsOf(entry));
53
54
  }
54
55
  for (const { artifact } of await this.#deps.artifacts()) {
55
56
  const uses = collectRefs(artifact.body);
@@ -77,7 +78,7 @@ export class Graph {
77
78
  const added = await this.projections.put(artifact.projections ?? [], persist);
78
79
  for (const id of added) {
79
80
  const entry = this.projections.get(id);
80
- this.index.addProjection(id, entry.deps, childProjections(entry));
81
+ this.index.addProjection(id, entry.deps, childProjections(entry), rowsOf(entry));
81
82
  }
82
83
  const uses = collectRefs(artifact.body);
83
84
  if (hasRefs(uses))
@@ -118,6 +119,24 @@ export class Graph {
118
119
  this.index.removeProjections(projections);
119
120
  return { records: records.sort(), projections, artifacts };
120
121
  }
122
+ /**
123
+ * Applies a change to the rows of dataset `dataset`. A page or projection that references a
124
+ * row holds its id, so a put needs no repair: the row resolves again at the next render, and
125
+ * only the pages that reference it are reported as touched. A deleted row leaves the pages that
126
+ * reference it unresolvable, so they and the projections above them are dropped. Nothing
127
+ * else is looked at.
128
+ */
129
+ async rowsChanged(dataset, puts, deletes) {
130
+ const id = (key) => `${dataset}/${key}`;
131
+ const touched = this.index.rowsReaching([...puts, ...deletes].map(id));
132
+ const gone = this.index.rowsReaching(deletes.map(id));
133
+ await this.projections.drop(gone.projections);
134
+ this.index.removeProjections(gone.projections);
135
+ return {
136
+ touched,
137
+ dropped: { records: [], projections: gone.projections, artifacts: gone.artifacts }
138
+ };
139
+ }
121
140
  /** forgets everything in memory and in storage */
122
141
  async clear() {
123
142
  this.records.forget();
@@ -125,15 +144,48 @@ export class Graph {
125
144
  this.index.clear();
126
145
  await this.#deps.storage.clearGraph();
127
146
  }
147
+ /** the datasets whose rows the artifact references, directly or through projections it reaches */
148
+ datasetsOf(artifact) {
149
+ const names = new Set();
150
+ const seen = new Set();
151
+ const queue = [artifact.body];
152
+ for (let node = queue.pop(); node !== undefined; node = queue.pop()) {
153
+ const refs = collectRefs(node);
154
+ for (const id of refs.rows ?? [])
155
+ names.add(splitRowId(id)?.dataset ?? id);
156
+ for (const id of refs.projections) {
157
+ if (seen.has(id))
158
+ continue;
159
+ seen.add(id);
160
+ const entry = this.projections.get(id);
161
+ if (entry)
162
+ queue.push(entry.data);
163
+ }
164
+ }
165
+ return [...names];
166
+ }
128
167
  /** the artifact with every reference replaced by what the graph holds */
129
168
  async materialize(artifact) {
130
169
  if (artifact.strategy === 'static' || !hasRefs(collectRefs(artifact.body))) {
131
170
  return { ok: true, artifact };
132
171
  }
133
172
  await this.ensure();
173
+ const datasets = this.datasetsOf(artifact);
174
+ if (datasets.length > 0) {
175
+ await this.#deps.loadDatasets?.(datasets);
176
+ const missing = datasets.filter((name) => !this.#deps.held?.(name));
177
+ if (missing.length > 0) {
178
+ return {
179
+ ok: false,
180
+ reason: `dataset ${missing[0]} is not held`,
181
+ datasets: missing
182
+ };
183
+ }
184
+ }
134
185
  const lookup = {
135
186
  record: (id) => this.records.get(id),
136
- projection: (id) => this.projections.get(id)
187
+ projection: (id) => this.projections.get(id),
188
+ ...(this.#deps.row ? { row: this.#deps.row } : {})
137
189
  };
138
190
  if (artifact.strategy !== 'assembly') {
139
191
  const data = resolveRefs(artifact.body.data, lookup);
@@ -6,14 +6,26 @@ import type { RefSet } from '../../protocol/graph.js';
6
6
  */
7
7
  export declare class DependencyIndex {
8
8
  #private;
9
- /** records that `projection` reads the tags in `deps` and references the projections in `children` */
10
- addProjection(projection: string, deps: readonly string[], children: readonly string[]): void;
9
+ /**
10
+ * records that `projection` reads the tags in `deps`, references the projections in `children`
11
+ * and the dataset rows in `rows`
12
+ */
13
+ addProjection(projection: string, deps: readonly string[], children: readonly string[], rows?: readonly string[]): void;
11
14
  removeProjections(ids: readonly string[]): void;
12
- /** records that `artifact` references these records and projections */
15
+ /** records that `artifact` references these records, projections and dataset rows */
13
16
  addArtifact(artifact: string, uses: RefSet): void;
14
17
  removeArtifacts(ids: readonly string[]): void;
15
18
  /** forgets everything */
16
19
  clear(): void;
20
+ /**
21
+ * What a change to dataset rows reaches: the projections that reference one directly, those
22
+ * and every projection above them, and the artifacts that use a reached projection or one of
23
+ * the rows. Row ids are `<dataset>/<key>`.
24
+ */
25
+ rowsReaching(rows: Iterable<string>): {
26
+ projections: string[];
27
+ artifacts: string[];
28
+ };
17
29
  /** the projections that read any of `tags` directly */
18
30
  projectionsReading(tags: Iterable<string>): string[];
19
31
  /** `projections` and every projection that references one of them, directly or not */