@drupflare/spool 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/README.md +499 -43
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/activity.d.ts +37 -0
  6. package/dist/client/activity.js +114 -0
  7. package/dist/client/entry.js +34 -1
  8. package/dist/client/index.d.ts +1 -0
  9. package/dist/client/index.js +1 -0
  10. package/dist/client/register.d.ts +23 -1
  11. package/dist/client/register.js +53 -7
  12. package/dist/media/embeds.d.ts +3 -2
  13. package/dist/media/embeds.js +16 -1
  14. package/dist/protocol/config.d.ts +16 -1
  15. package/dist/protocol/config.js +3 -1
  16. package/dist/protocol/dataset.d.ts +183 -0
  17. package/dist/protocol/dataset.js +206 -0
  18. package/dist/protocol/envelope.d.ts +3 -0
  19. package/dist/protocol/envelope.js +14 -3
  20. package/dist/protocol/errors.d.ts +1 -1
  21. package/dist/protocol/graph.d.ts +23 -4
  22. package/dist/protocol/graph.js +34 -3
  23. package/dist/protocol/hash.d.ts +2 -0
  24. package/dist/protocol/hash.js +8 -0
  25. package/dist/protocol/index.d.ts +7 -0
  26. package/dist/protocol/index.js +7 -0
  27. package/dist/protocol/inline.d.ts +14 -0
  28. package/dist/protocol/inline.js +43 -0
  29. package/dist/protocol/navigation.d.ts +81 -0
  30. package/dist/protocol/navigation.js +249 -0
  31. package/dist/protocol/normalise.d.ts +4 -3
  32. package/dist/protocol/normalise.js +6 -4
  33. package/dist/protocol/plan.d.ts +100 -0
  34. package/dist/protocol/plan.js +661 -0
  35. package/dist/protocol/provision.d.ts +52 -1
  36. package/dist/protocol/provision.js +53 -5
  37. package/dist/protocol/query.d.ts +141 -0
  38. package/dist/protocol/query.js +252 -0
  39. package/dist/protocol/route.d.ts +85 -5
  40. package/dist/protocol/route.js +91 -0
  41. package/dist/protocol/write.d.ts +146 -0
  42. package/dist/protocol/write.js +101 -0
  43. package/dist/proxy/bases.d.ts +53 -0
  44. package/dist/proxy/bases.js +76 -0
  45. package/dist/proxy/index.d.ts +1 -0
  46. package/dist/proxy/index.js +1 -0
  47. package/dist/proxy/proxy.js +8 -2
  48. package/dist/report/capacity.d.ts +59 -0
  49. package/dist/report/capacity.js +76 -0
  50. package/dist/report/channels.d.ts +60 -0
  51. package/dist/report/channels.js +65 -0
  52. package/dist/report/dataset.d.ts +48 -0
  53. package/dist/report/dataset.js +93 -0
  54. package/dist/report/derive.d.ts +285 -0
  55. package/dist/report/derive.js +676 -1
  56. package/dist/report/economics.d.ts +11 -1
  57. package/dist/report/economics.js +19 -1
  58. package/dist/report/index.d.ts +5 -0
  59. package/dist/report/index.js +5 -0
  60. package/dist/report/inputs.d.ts +97 -0
  61. package/dist/report/inputs.js +133 -0
  62. package/dist/report/institution.d.ts +284 -0
  63. package/dist/report/institution.js +225 -0
  64. package/dist/report/ledger.d.ts +28 -0
  65. package/dist/report/ledger.js +30 -0
  66. package/dist/report/loops.d.ts +73 -0
  67. package/dist/report/loops.js +101 -0
  68. package/dist/report/measurement.d.ts +1 -1
  69. package/dist/report/measurement.js +6 -1
  70. package/dist/report/model.d.ts +2 -0
  71. package/dist/report/model.js +4 -0
  72. package/dist/report/thirdparty.d.ts +57 -0
  73. package/dist/report/thirdparty.js +111 -0
  74. package/dist/report/traffic.d.ts +11 -2
  75. package/dist/report/traffic.js +13 -3
  76. package/dist/report/trim.d.ts +56 -0
  77. package/dist/report/trim.js +83 -1
  78. package/dist/server/dataset.d.ts +48 -0
  79. package/dist/server/dataset.js +308 -0
  80. package/dist/server/embedded.d.ts +3 -3
  81. package/dist/server/embedded.js +3 -3
  82. package/dist/server/index.d.ts +3 -0
  83. package/dist/server/index.js +3 -0
  84. package/dist/server/spool.d.ts +40 -0
  85. package/dist/server/spool.js +184 -23
  86. package/dist/server/write.d.ts +27 -0
  87. package/dist/server/write.js +127 -0
  88. package/dist/sw/analytics.d.ts +6 -0
  89. package/dist/sw/analytics.js +14 -0
  90. package/dist/sw/dataset-sync.d.ts +76 -0
  91. package/dist/sw/dataset-sync.js +149 -0
  92. package/dist/sw/dataset.d.ts +91 -0
  93. package/dist/sw/dataset.js +318 -0
  94. package/dist/sw/graph/index.d.ts +31 -1
  95. package/dist/sw/graph/index.js +57 -5
  96. package/dist/sw/graph/invalidate.d.ts +15 -3
  97. package/dist/sw/graph/invalidate.js +42 -4
  98. package/dist/sw/idb.js +78 -9
  99. package/dist/sw/partition.d.ts +6 -1
  100. package/dist/sw/partition.js +21 -2
  101. package/dist/sw/principal-dataset.d.ts +64 -0
  102. package/dist/sw/principal-dataset.js +254 -0
  103. package/dist/sw/render.d.ts +8 -1
  104. package/dist/sw/render.js +28 -1
  105. package/dist/sw/replica.d.ts +14 -3
  106. package/dist/sw/replica.js +46 -5
  107. package/dist/sw/router.js +2 -2
  108. package/dist/sw/runtime.d.ts +43 -2
  109. package/dist/sw/runtime.js +440 -29
  110. package/dist/sw/seal.d.ts +13 -0
  111. package/dist/sw/seal.js +13 -4
  112. package/dist/sw/static.d.ts +18 -0
  113. package/dist/sw/static.js +84 -1
  114. package/dist/sw/storage.d.ts +51 -1
  115. package/dist/sw/storage.js +33 -1
  116. package/dist/sw/write.d.ts +107 -0
  117. package/dist/sw/write.js +442 -0
  118. package/package.json +5 -2
@@ -0,0 +1,149 @@
1
+ import { PATHS } from '../protocol/config.js';
2
+ import { encodeDatasetRequest, parseDatasetResponse } from '../protocol/dataset.js';
3
+ import { HEADERS } from '../protocol/envelope.js';
4
+ import { ProtocolError } from '../protocol/errors.js';
5
+ /**
6
+ * The guard of a principal dataset held in `partition`: `401` ends the session, `403` revokes the
7
+ * dataset, and a success that does not name exactly this partition is another principal's or none.
8
+ */
9
+ export function principalGuard(partition) {
10
+ return (response) => {
11
+ if (response.status === 401)
12
+ return 'ended';
13
+ if (response.status === 403)
14
+ return 'revoked';
15
+ if (response.ok && response.headers.get(HEADERS.partition) !== partition)
16
+ return 'partition';
17
+ return 'ok';
18
+ };
19
+ }
20
+ /** seed pages followed before a sync gives up on a server whose paging does not end */
21
+ export const DATASET_MAX_PAGES = 1000;
22
+ class Refused extends Error {
23
+ reason;
24
+ by;
25
+ constructor(reason, by) {
26
+ super(reason);
27
+ this.reason = reason;
28
+ this.by = by;
29
+ }
30
+ }
31
+ /**
32
+ * Brings dataset `name` current. A held dataset asks for the changes since its cursor; one that
33
+ * is not held, or that the server resets (past its horizon, another schema), seeds in pages. The
34
+ * rows are applied to a staging area and written in one transaction after the collection proof
35
+ * matches. A proof that fails after a seed is repaired by one patch from the first page's cursor,
36
+ * which covers rows that moved while the pages were read; a proof that still fails clears the
37
+ * dataset and reseeds once, and a second failure leaves it not held. Never throws: every end is
38
+ * an outcome.
39
+ */
40
+ export async function syncDataset(deps, name) {
41
+ const { store, origin } = deps;
42
+ const epoch = deps.epoch();
43
+ let requests = 0;
44
+ const get = async (request) => {
45
+ requests += 1;
46
+ const response = await deps.fetch(`${origin}${PATHS.dataset}?${encodeDatasetRequest(request)}`, { cache: 'no-store', credentials: 'same-origin' });
47
+ const verdict = deps.guard?.(response) ?? 'ok';
48
+ if (verdict !== 'ok')
49
+ throw new Refused(verdict);
50
+ if (response.status === 409)
51
+ throw new Refused('declined', response.headers.get(HEADERS.provisionReason));
52
+ if (response.status === 400 || response.status === 404)
53
+ throw new Refused('unavailable');
54
+ if (!response.ok)
55
+ throw new TypeError(`dataset answered ${response.status}`);
56
+ const parsed = parseDatasetResponse(await response.json());
57
+ if (parsed.name !== name)
58
+ throw new Refused('bad-response');
59
+ return parsed;
60
+ };
61
+ const write = async (staging, at, status) => {
62
+ try {
63
+ if (deps.epoch() !== epoch)
64
+ throw new Refused('epoch');
65
+ if (!(await store.commit(staging, at, epoch)))
66
+ throw new Refused('epoch');
67
+ }
68
+ catch (error) {
69
+ if (error instanceof Refused)
70
+ throw error;
71
+ throw new Refused('storage');
72
+ }
73
+ return { name, requests, status, cursor: at.cursor, rows: staging.rows.size };
74
+ };
75
+ const seed = async (first, status, retry) => {
76
+ let page = first ?? (await get({ name }));
77
+ const { cursor, schema } = page;
78
+ const budget = await deps.budget();
79
+ const staging = store.stage(name);
80
+ let matches;
81
+ let after;
82
+ for (let pages = 1;; pages += 1) {
83
+ if (deps.epoch() !== epoch)
84
+ throw new Refused('epoch');
85
+ matches = await staging.add(page);
86
+ if (budget !== undefined && staging.bytes > budget)
87
+ throw new Refused('budget');
88
+ if (page.next === undefined)
89
+ break;
90
+ if (pages >= DATASET_MAX_PAGES || (after !== undefined && page.next <= after)) {
91
+ throw new Refused('bad-response');
92
+ }
93
+ after = page.next;
94
+ page = await get({ name, after });
95
+ }
96
+ if (matches === true)
97
+ return write(staging, { cursor, schema }, status);
98
+ // rows moved while the pages were read: the changes since the first page repair the copy
99
+ const patch = await get({ name, since: cursor, schema });
100
+ if (!patch.reset && (await staging.add(patch)) === true) {
101
+ return write(staging, { cursor: patch.cursor, schema }, status);
102
+ }
103
+ if (retry)
104
+ return seed(undefined, 'reseeded', false);
105
+ throw new Refused('proof');
106
+ };
107
+ try {
108
+ const held = await store.get(name);
109
+ if (!held)
110
+ return await seed(undefined, 'seeded', true);
111
+ const { cursor, schema, digest, count } = held.meta;
112
+ const response = await get({ name, since: cursor, schema });
113
+ if (response.reset)
114
+ return await seed(response, 'reseeded', true);
115
+ if (response.cursor < cursor)
116
+ return { name, requests, status: 'current', cursor, rows: count };
117
+ const proof = response.proof;
118
+ if (response.cursor === cursor) {
119
+ // nothing moved; a proof that disagrees with the held rows means the copy is damaged
120
+ if (proof && (proof.count !== count || proof.digest !== digest)) {
121
+ await store.drop(name);
122
+ return await seed(undefined, 'reseeded', false);
123
+ }
124
+ if (!(await store.touch(name, epoch)))
125
+ throw new Refused('epoch');
126
+ return { name, requests, status: 'current', cursor, rows: count };
127
+ }
128
+ const staging = store.stage(name, held);
129
+ if ((await staging.add(response)) === true) {
130
+ return await write(staging, { cursor: response.cursor, schema }, 'patched');
131
+ }
132
+ // the patched rows do not match the server's proof: nothing wrong is kept
133
+ await store.drop(name);
134
+ return await seed(undefined, 'reseeded', false);
135
+ }
136
+ catch (error) {
137
+ if (error instanceof Refused) {
138
+ // the server stopped admitting this principal to the dataset, so no row of it stays
139
+ if (error.reason === 'revoked' && deps.epoch() === epoch) {
140
+ await store.drop(name).catch(() => undefined);
141
+ }
142
+ return { name, requests, status: 'refused', reason: error.reason, by: error.by };
143
+ }
144
+ if (error instanceof ProtocolError) {
145
+ return { name, requests, status: 'refused', reason: 'bad-response' };
146
+ }
147
+ return { name, requests, status: 'offline' };
148
+ }
149
+ }
@@ -0,0 +1,91 @@
1
+ import { type DatasetResponse, type DatasetRow } from '../protocol/dataset.js';
2
+ import { type PlanTables } from '../protocol/plan.js';
3
+ import type { DatasetMeta, DatasetRowRecord, ReplicaStorage } from './storage.js';
4
+ /** bytes of a row's JSON, the unit `maxBytes` and the storage budget count in */
5
+ export declare const rowBytes: (row: DatasetRow) => number;
6
+ /** a dataset in memory, as a verified load or a commit left it */
7
+ export type Held = {
8
+ meta: DatasetMeta;
9
+ rows: Map<string, DatasetRow>;
10
+ /** the hash of every row, so a patch never hashes a row it replaces or deletes */
11
+ hashes: Map<string, string>;
12
+ };
13
+ /** what the store needs from the replica around it */
14
+ export type DatasetStoreDeps = {
15
+ storage: ReplicaStorage;
16
+ now(): number;
17
+ /** the replica's wipe count; a commit that began under another one is dropped */
18
+ epoch(): number;
19
+ /** the row hash; {@link rowHash}. A test counts calls through it */
20
+ hash?: (key: string, row: DatasetRow) => Promise<string>;
21
+ /** told after every commit which row keys it put and which it removed */
22
+ onChange?(name: string, puts: string[], deletes: string[]): Promise<void> | void;
23
+ };
24
+ /**
25
+ * One dataset's state while a response or a run of seed pages is applied, before anything is
26
+ * written. A staged seed replaces the held rows; a staged patch changes them. Nothing here
27
+ * touches storage, so a failed or dropped apply leaves the store as it was.
28
+ */
29
+ export declare class Staging {
30
+ #private;
31
+ readonly name: string;
32
+ /** whether the commit replaces every held row */
33
+ clear: boolean;
34
+ rows: Map<string, DatasetRow>;
35
+ hashes: Map<string, string>;
36
+ digest: string;
37
+ bytes: number;
38
+ constructor(deps: DatasetStoreDeps, name: string, base?: Held);
39
+ /** applies a response; returns whether the result equals the response's proof, null without one */
40
+ add(response: DatasetResponse): Promise<boolean | null>;
41
+ /** the rows a commit writes: all of them for a replacement, the touched ones for a patch */
42
+ writes(): {
43
+ puts: DatasetRowRecord[];
44
+ deletes: string[];
45
+ };
46
+ }
47
+ /**
48
+ * The datasets a device holds. Rows live once in memory after a verified load and once in
49
+ * storage, and a response is applied to a {@link Staging} first and committed in one storage
50
+ * transaction. Reads and writes of the held set go through one queue, so a commit and a wipe
51
+ * never interleave.
52
+ */
53
+ export declare class DatasetStore {
54
+ #private;
55
+ /** datasets found damaged on a load and cleared, since the worker started */
56
+ evicted: number;
57
+ constructor(deps: DatasetStoreDeps);
58
+ /** the meta of every held dataset */
59
+ list(): Promise<DatasetMeta[]>;
60
+ /**
61
+ * Loads one dataset into memory and checks it: the stored row count and the sum of the stored row
62
+ * hashes must equal the meta. A dataset that fails, which is what a partly evicted store looks
63
+ * like, is cleared and reads as absent. No row is hashed.
64
+ */
65
+ get(name: string): Promise<Held | undefined>;
66
+ /** the held rows in ascending key order, or undefined when `name` is not loaded and held */
67
+ rows(name: string): Promise<DatasetRow[] | undefined>;
68
+ /** the base table and each joined table a plan reads, or undefined when any of them is not held */
69
+ tables(base: string, joins: readonly string[]): Promise<PlanTables | undefined>;
70
+ /** whether `name` is held and in memory */
71
+ loaded(name: string): boolean;
72
+ /** a held row, looked up in memory; undefined unless {@link DatasetStore.get} loaded the dataset */
73
+ row(name: string, key: string): DatasetRow | undefined;
74
+ /** a staging area over the held rows of `name`, or an empty one for a seed */
75
+ stage(name: string, base?: Held): Staging;
76
+ /**
77
+ * Writes a staged apply. `cursor` and `schema` are what the response brought the rows to.
78
+ * Returns false, writing nothing, when the replica was wiped since `epoch` was read. Rejects,
79
+ * with the held rows and the memory copy untouched, when the storage transaction fails.
80
+ */
81
+ commit(staging: Staging, at: {
82
+ cursor: number;
83
+ schema: string;
84
+ }, epoch: number): Promise<boolean>;
85
+ /** records that a sync found the held rows current, with no change to them */
86
+ touch(name: string, epoch: number): Promise<boolean>;
87
+ /** clears one dataset's rows and meta, and nothing else */
88
+ drop(name: string): Promise<void>;
89
+ /** clears every dataset, for a wipe of the replica */
90
+ clear(): Promise<void>;
91
+ }
@@ -0,0 +1,318 @@
1
+ import { applyChanges } from '../protocol/dataset.js';
2
+ import { compareText, TableIndex } from '../protocol/plan.js';
3
+ import { digestAdd, EMPTY_DIGEST } from '../protocol/query.js';
4
+ const encoder = new TextEncoder();
5
+ /** bytes of a row's JSON, the unit `maxBytes` and the storage budget count in */
6
+ export const rowBytes = (row) => encoder.encode(JSON.stringify(row)).length;
7
+ /**
8
+ * One dataset's state while a response or a run of seed pages is applied, before anything is
9
+ * written. A staged seed replaces the held rows; a staged patch changes them. Nothing here
10
+ * touches storage, so a failed or dropped apply leaves the store as it was.
11
+ */
12
+ export class Staging {
13
+ name;
14
+ /** whether the commit replaces every held row */
15
+ clear;
16
+ rows;
17
+ hashes;
18
+ digest;
19
+ bytes;
20
+ #deps;
21
+ #puts = new Set();
22
+ #deletes = new Set();
23
+ constructor(deps, name, base) {
24
+ this.#deps = deps;
25
+ this.name = name;
26
+ this.clear = base === undefined;
27
+ this.rows = base?.rows ?? new Map();
28
+ this.hashes = base?.hashes ?? new Map();
29
+ this.digest = base?.meta.digest ?? EMPTY_DIGEST;
30
+ this.bytes = base?.meta.bytes ?? 0;
31
+ }
32
+ /** applies a response; returns whether the result equals the response's proof, null without one */
33
+ async add(response) {
34
+ const fresh = response.reset && response.after === undefined;
35
+ if (fresh) {
36
+ this.clear = true;
37
+ this.bytes = 0;
38
+ this.#puts.clear();
39
+ this.#deletes.clear();
40
+ }
41
+ // bytes follow the rows in the order the response applies them, deletes first
42
+ const overlay = new Map();
43
+ const current = (key) => overlay.has(key) ? overlay.get(key) : fresh ? undefined : this.rows.get(key);
44
+ for (const { key } of response.deletes) {
45
+ const old = current(key);
46
+ if (old !== undefined)
47
+ this.bytes -= rowBytes(old);
48
+ overlay.set(key, undefined);
49
+ }
50
+ for (const { key, row } of response.puts) {
51
+ const old = current(key);
52
+ if (old !== undefined)
53
+ this.bytes -= rowBytes(old);
54
+ this.bytes += rowBytes(row);
55
+ overlay.set(key, row);
56
+ }
57
+ const options = {
58
+ hashes: this.hashes,
59
+ ...(this.#deps.hash ? { hash: this.#deps.hash } : {})
60
+ };
61
+ const applied = await applyChanges(this.rows, response, this.digest, options);
62
+ this.rows = applied.rows;
63
+ this.hashes = applied.hashes;
64
+ this.digest = applied.digest;
65
+ for (const { key } of response.deletes) {
66
+ this.#deletes.add(key);
67
+ this.#puts.delete(key);
68
+ }
69
+ for (const { key } of response.puts) {
70
+ this.#puts.add(key);
71
+ this.#deletes.delete(key);
72
+ }
73
+ return applied.matches;
74
+ }
75
+ /** the rows a commit writes: all of them for a replacement, the touched ones for a patch */
76
+ writes() {
77
+ const keys = this.clear ? [...this.rows.keys()] : [...this.#puts];
78
+ return {
79
+ puts: keys.map((key) => ({
80
+ name: this.name,
81
+ key,
82
+ row: this.rows.get(key),
83
+ hash: this.hashes.get(key)
84
+ })),
85
+ deletes: this.clear ? [] : [...this.#deletes]
86
+ };
87
+ }
88
+ }
89
+ /**
90
+ * The datasets a device holds. Rows live once in memory after a verified load and once in
91
+ * storage, and a response is applied to a {@link Staging} first and committed in one storage
92
+ * transaction. Reads and writes of the held set go through one queue, so a commit and a wipe
93
+ * never interleave.
94
+ */
95
+ export class DatasetStore {
96
+ #deps;
97
+ #held = new Map();
98
+ #loaded = new Map();
99
+ #metas;
100
+ #queue = Promise.resolve();
101
+ #sorted = new Map();
102
+ /** the key-ordered tables and their indexes a plan reads; dropped on a commit, a drop or a wipe */
103
+ #tables = new Map();
104
+ /** datasets found damaged on a load and cleared, since the worker started */
105
+ evicted = 0;
106
+ constructor(deps) {
107
+ this.#deps = deps;
108
+ }
109
+ #serial(fn) {
110
+ const run = this.#queue.then(fn, fn);
111
+ this.#queue = run.catch(() => undefined);
112
+ return run;
113
+ }
114
+ #metaMap() {
115
+ this.#metas ??= this.#deps.storage
116
+ .getDatasets()
117
+ .then((all) => new Map(all.map((meta) => [meta.name, meta])));
118
+ return this.#metas;
119
+ }
120
+ /** the meta of every held dataset */
121
+ async list() {
122
+ return [...(await this.#metaMap()).values()].sort((a, b) => (a.name < b.name ? -1 : 1));
123
+ }
124
+ /**
125
+ * Loads one dataset into memory and checks it: the stored row count and the sum of the stored row
126
+ * hashes must equal the meta. A dataset that fails, which is what a partly evicted store looks
127
+ * like, is cleared and reads as absent. No row is hashed.
128
+ */
129
+ get(name) {
130
+ let held = this.#held.get(name);
131
+ if (!held) {
132
+ held = this.#load(name);
133
+ this.#held.set(name, held);
134
+ }
135
+ return held;
136
+ }
137
+ async #load(name) {
138
+ const meta = (await this.#metaMap()).get(name);
139
+ if (!meta)
140
+ return undefined;
141
+ const records = await this.#deps.storage.readDataset(name);
142
+ let sum = EMPTY_DIGEST;
143
+ let sound = records.length === meta.count;
144
+ try {
145
+ for (const record of records)
146
+ sum = digestAdd(sum, record.hash);
147
+ }
148
+ catch {
149
+ sound = false;
150
+ }
151
+ if (!sound || sum !== meta.digest) {
152
+ this.evicted += 1;
153
+ await this.drop(name);
154
+ return undefined;
155
+ }
156
+ const held = {
157
+ meta,
158
+ rows: new Map(records.map((r) => [r.key, r.row])),
159
+ hashes: new Map(records.map((r) => [r.key, r.hash]))
160
+ };
161
+ this.#loaded.set(name, held);
162
+ return held;
163
+ }
164
+ /** the held rows in ascending key order, or undefined when `name` is not loaded and held */
165
+ async rows(name) {
166
+ const held = await this.get(name);
167
+ if (!held)
168
+ return undefined;
169
+ let sorted = this.#sorted.get(name);
170
+ if (!sorted) {
171
+ sorted = [...held.rows.keys()].sort().map((key) => held.rows.get(key));
172
+ this.#sorted.set(name, sorted);
173
+ }
174
+ return sorted;
175
+ }
176
+ /** the base table and each joined table a plan reads, or undefined when any of them is not held */
177
+ async tables(base, joins) {
178
+ const baseTable = await this.#table(base);
179
+ if (!baseTable)
180
+ return undefined;
181
+ const joined = new Map();
182
+ for (const name of joins) {
183
+ const table = await this.#table(name);
184
+ if (!table)
185
+ return undefined;
186
+ joined.set(name, table);
187
+ }
188
+ return { base: baseTable, joined: (name) => joined.get(name) };
189
+ }
190
+ async #table(name) {
191
+ const held = await this.get(name);
192
+ if (!held)
193
+ return undefined;
194
+ let table = this.#tables.get(name);
195
+ if (!table) {
196
+ const keys = [...held.rows.keys()].sort(compareText);
197
+ table = { keys, rows: held.rows, index: new TableIndex(keys, held.rows) };
198
+ this.#tables.set(name, table);
199
+ }
200
+ return table;
201
+ }
202
+ /** whether `name` is held and in memory */
203
+ loaded(name) {
204
+ return this.#loaded.has(name);
205
+ }
206
+ /** a held row, looked up in memory; undefined unless {@link DatasetStore.get} loaded the dataset */
207
+ row(name, key) {
208
+ return this.#loaded.get(name)?.rows.get(key);
209
+ }
210
+ /** a staging area over the held rows of `name`, or an empty one for a seed */
211
+ stage(name, base) {
212
+ return new Staging(this.#deps, name, base);
213
+ }
214
+ /**
215
+ * Writes a staged apply. `cursor` and `schema` are what the response brought the rows to.
216
+ * Returns false, writing nothing, when the replica was wiped since `epoch` was read. Rejects,
217
+ * with the held rows and the memory copy untouched, when the storage transaction fails.
218
+ */
219
+ commit(staging, at, epoch) {
220
+ return this.#serial(async () => {
221
+ if (this.#deps.epoch() !== epoch)
222
+ return false;
223
+ const before = this.#loaded.get(staging.name);
224
+ const meta = {
225
+ name: staging.name,
226
+ schema: at.schema,
227
+ cursor: at.cursor,
228
+ count: staging.rows.size,
229
+ digest: staging.digest,
230
+ bytes: staging.bytes,
231
+ appliedAt: this.#deps.now()
232
+ };
233
+ const { puts, deletes } = staging.writes();
234
+ await this.#deps.storage.commitDataset({
235
+ name: staging.name,
236
+ meta,
237
+ clear: staging.clear,
238
+ puts,
239
+ deletes
240
+ });
241
+ const held = { meta, rows: staging.rows, hashes: staging.hashes };
242
+ this.#tables.delete(staging.name);
243
+ await this.#set(staging.name, held);
244
+ await this.#deps.onChange?.(staging.name, ...changedKeys(before, held));
245
+ return true;
246
+ });
247
+ }
248
+ /** records that a sync found the held rows current, with no change to them */
249
+ touch(name, epoch) {
250
+ return this.#serial(async () => {
251
+ const held = this.#loaded.get(name);
252
+ if (!held || this.#deps.epoch() !== epoch)
253
+ return false;
254
+ const meta = { ...held.meta, appliedAt: this.#deps.now() };
255
+ await this.#deps.storage.commitDataset({
256
+ name,
257
+ meta,
258
+ clear: false,
259
+ puts: [],
260
+ deletes: []
261
+ });
262
+ await this.#set(name, { ...held, meta });
263
+ return true;
264
+ });
265
+ }
266
+ /** clears one dataset's rows and meta, and nothing else */
267
+ drop(name) {
268
+ return this.#serial(async () => {
269
+ const before = this.#loaded.get(name);
270
+ await this.#deps.storage.commitDataset({
271
+ name,
272
+ meta: null,
273
+ clear: true,
274
+ puts: [],
275
+ deletes: []
276
+ });
277
+ this.#tables.delete(name);
278
+ await this.#set(name, undefined);
279
+ if (before)
280
+ await this.#deps.onChange?.(name, [], [...before.rows.keys()]);
281
+ });
282
+ }
283
+ /** clears every dataset, for a wipe of the replica */
284
+ clear() {
285
+ return this.#serial(async () => {
286
+ await this.#deps.storage.clearDatasets();
287
+ this.#held.clear();
288
+ this.#loaded.clear();
289
+ this.#sorted.clear();
290
+ this.#tables.clear();
291
+ this.#metas = Promise.resolve(new Map());
292
+ });
293
+ }
294
+ async #set(name, held) {
295
+ const metas = await this.#metaMap();
296
+ this.#sorted.delete(name);
297
+ if (held) {
298
+ metas.set(name, held.meta);
299
+ this.#loaded.set(name, held);
300
+ this.#held.set(name, Promise.resolve(held));
301
+ }
302
+ else {
303
+ metas.delete(name);
304
+ this.#loaded.delete(name);
305
+ this.#held.delete(name);
306
+ }
307
+ }
308
+ }
309
+ /** the keys a commit put (new or different rows) and removed, by comparing row hashes */
310
+ function changedKeys(before, after) {
311
+ const puts = [];
312
+ for (const [key, hash] of after.hashes) {
313
+ if (before?.hashes.get(key) !== hash)
314
+ puts.push(key);
315
+ }
316
+ const deletes = [...(before?.hashes.keys() ?? [])].filter((key) => !after.hashes.has(key));
317
+ return [puts, deletes];
318
+ }
@@ -11,13 +11,17 @@ export type Invalidated = {
11
11
  projections: string[];
12
12
  artifacts: string[];
13
13
  };
14
- /** a page with every reference resolved, or why it cannot be built from what is held */
14
+ /**
15
+ * a page with every reference resolved, or why it cannot be built from what is held; `datasets`
16
+ * names the datasets the page references that the device does not hold, which a seed would fix
17
+ */
15
18
  export type Materialized = {
16
19
  ok: true;
17
20
  artifact: Artifact;
18
21
  } | {
19
22
  ok: false;
20
23
  reason: string;
24
+ datasets?: string[];
21
25
  };
22
26
  /** what the graph reads from the replica around it */
23
27
  export type GraphDeps = {
@@ -26,6 +30,22 @@ export type GraphDeps = {
26
30
  tagVersion(tag: string): number;
27
31
  /** every stored artifact, read once to build the index on first use */
28
32
  artifacts(): Promise<StoredArtifact[]>;
33
+ /** the held row `key` of `dataset`; undefined when it is not held or its dataset is not loaded */
34
+ row?(dataset: string, key: string): unknown;
35
+ /** whether `dataset` is held and loaded */
36
+ held?(dataset: string): boolean;
37
+ /** loads these datasets into memory before a page that references their rows is resolved */
38
+ loadDatasets?(names: string[]): Promise<void>;
39
+ };
40
+ /** what a change to dataset rows did to the graph */
41
+ export type RowEffect = {
42
+ /** projections and artifacts that reference a changed row; they resolve it again at render, so nothing is fetched */
43
+ touched: {
44
+ projections: string[];
45
+ artifacts: string[];
46
+ };
47
+ /** what a deleted row left unresolvable, dropped from the graph; the replica drops the artifacts */
48
+ dropped: Invalidated;
29
49
  };
30
50
  /** whether an artifact references anything in the graph */
31
51
  export declare function usesGraph(artifact: Artifact): boolean;
@@ -60,8 +80,18 @@ export declare class Graph {
60
80
  * was. Artifacts that reference anything dropped are reported, not removed: the replica owns them.
61
81
  */
62
82
  invalidate(changed: Readonly<Record<string, number>>): Promise<Invalidated>;
83
+ /**
84
+ * Applies a change to the rows of dataset `dataset`. A page or projection that references a
85
+ * row holds its id, so a put needs no repair: the row resolves again at the next render, and
86
+ * only the pages that reference it are reported as touched. A deleted row leaves the pages that
87
+ * reference it unresolvable, so they and the projections above them are dropped. Nothing
88
+ * else is looked at.
89
+ */
90
+ rowsChanged(dataset: string, puts: readonly string[], deletes: readonly string[]): Promise<RowEffect>;
63
91
  /** forgets everything in memory and in storage */
64
92
  clear(): Promise<void>;
93
+ /** the datasets whose rows the artifact references, directly or through projections it reaches */
94
+ datasetsOf(artifact: Artifact): string[];
65
95
  /** the artifact with every reference replaced by what the graph holds */
66
96
  materialize(artifact: Artifact): Promise<Materialized>;
67
97
  }