@drupflare/spool 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +263 -41
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/entry.js +33 -1
- package/dist/client/register.d.ts +23 -1
- package/dist/client/register.js +53 -7
- package/dist/protocol/config.d.ts +16 -1
- package/dist/protocol/config.js +3 -1
- package/dist/protocol/dataset.d.ts +183 -0
- package/dist/protocol/dataset.js +206 -0
- package/dist/protocol/envelope.d.ts +1 -0
- package/dist/protocol/envelope.js +2 -1
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/index.d.ts +3 -0
- package/dist/protocol/index.js +3 -0
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +46 -0
- package/dist/protocol/provision.js +38 -0
- package/dist/protocol/query.d.ts +66 -5
- package/dist/protocol/query.js +103 -4
- package/dist/protocol/route.d.ts +6 -2
- package/dist/protocol/route.js +31 -3
- package/dist/protocol/write.d.ts +146 -0
- package/dist/protocol/write.js +101 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +11 -1
- package/dist/report/economics.js +19 -1
- package/dist/report/index.d.ts +2 -0
- package/dist/report/index.js +2 -0
- package/dist/report/inputs.d.ts +97 -0
- package/dist/report/inputs.js +133 -0
- package/dist/report/institution.d.ts +188 -1
- package/dist/report/institution.js +146 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +3 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/dataset.d.ts +48 -0
- package/dist/server/dataset.js +308 -0
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +2 -0
- package/dist/server/spool.d.ts +20 -0
- package/dist/server/spool.js +126 -18
- package/dist/server/write.d.ts +27 -0
- package/dist/server/write.js +127 -0
- package/dist/sw/analytics.d.ts +6 -0
- package/dist/sw/analytics.js +14 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.js +78 -9
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/render.d.ts +7 -2
- package/dist/sw/render.js +19 -5
- package/dist/sw/replica.d.ts +12 -3
- package/dist/sw/replica.js +40 -5
- package/dist/sw/runtime.d.ts +43 -2
- package/dist/sw/runtime.js +340 -24
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +107 -0
- package/dist/sw/write.js +442 -0
- package/package.json +3 -2
package/dist/client/register.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { PROTOCOL_VERSION } from '../protocol/artifact.js';
|
|
2
2
|
import { PATHS } from '../protocol/config.js';
|
|
3
3
|
import { closeOnPageHide } from './lifecycle.js';
|
|
4
|
+
/** how long a flush may take to answer; the requests it sends are the wait */
|
|
5
|
+
const FLUSH_WAIT_MS = 30_000;
|
|
4
6
|
function ask(worker, message, timeoutMs) {
|
|
5
7
|
return new Promise((resolve, reject) => {
|
|
6
8
|
const channel = new MessageChannel();
|
|
@@ -26,9 +28,35 @@ export async function register(options = {}) {
|
|
|
26
28
|
if (!container)
|
|
27
29
|
return null;
|
|
28
30
|
const timeout = options.helloTimeoutMs ?? 5000;
|
|
31
|
+
let granted = null;
|
|
32
|
+
let asked = false;
|
|
33
|
+
let worker;
|
|
34
|
+
// persist() is Window-only, so the worker asks a page once after it first holds a dataset; the
|
|
35
|
+
// grant covers IndexedDB the worker writes. Nothing waits on the answer.
|
|
36
|
+
const persist = async () => {
|
|
37
|
+
if (asked)
|
|
38
|
+
return;
|
|
39
|
+
asked = true;
|
|
40
|
+
const storage = options.storage ?? navigator.storage;
|
|
41
|
+
try {
|
|
42
|
+
granted =
|
|
43
|
+
(await storage?.persisted?.()) === true || (await storage?.persist?.()) === true;
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
granted = false;
|
|
47
|
+
}
|
|
48
|
+
(options.target ?? window).dispatchEvent(new CustomEvent('spool:persisted', { detail: { granted } }));
|
|
49
|
+
(worker ?? container.controller)?.postMessage({ type: 'spool:persisted', granted });
|
|
50
|
+
};
|
|
29
51
|
// the worker tells every page when it wipes the principal store, so a page can blank what it shows
|
|
30
52
|
container.addEventListener?.('message', (event) => {
|
|
31
53
|
const data = event.data;
|
|
54
|
+
if (data?.type === 'spool:persist')
|
|
55
|
+
void persist();
|
|
56
|
+
if (data?.type === 'spool:written') {
|
|
57
|
+
const { queued, flushed, conflicted } = data;
|
|
58
|
+
(options.target ?? window).dispatchEvent(new CustomEvent('spool:written', { detail: { queued, flushed, conflicted } }));
|
|
59
|
+
}
|
|
32
60
|
if (data?.type !== 'spool:cleared')
|
|
33
61
|
return;
|
|
34
62
|
(options.target ?? window).dispatchEvent(new CustomEvent('spool:cleared', { detail: { reason: data.reason } }));
|
|
@@ -38,21 +66,39 @@ export async function register(options = {}) {
|
|
|
38
66
|
updateViaCache: 'all'
|
|
39
67
|
});
|
|
40
68
|
const registration = await container.ready;
|
|
41
|
-
const
|
|
42
|
-
if (!
|
|
69
|
+
const active = registration.active;
|
|
70
|
+
if (!active)
|
|
43
71
|
throw new Error('the worker is not active');
|
|
44
|
-
|
|
72
|
+
worker = active;
|
|
73
|
+
const hello = await ask(active, { type: 'spool:hello', protocol: PROTOCOL_VERSION }, timeout);
|
|
74
|
+
// a write call never throws: a worker that is gone or silent is a refusal like any other
|
|
75
|
+
const askWrite = async (message, waitMs = timeout) => {
|
|
76
|
+
try {
|
|
77
|
+
const reply = await ask(registration.active ?? active, message, waitMs);
|
|
78
|
+
if (reply?.type === message.type)
|
|
79
|
+
return reply.result;
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
// falls through to the refusal
|
|
83
|
+
}
|
|
84
|
+
return { ok: false, code: 'unavailable', message: 'the worker did not answer' };
|
|
85
|
+
};
|
|
45
86
|
return {
|
|
46
87
|
registration,
|
|
47
88
|
hello,
|
|
48
|
-
status: () => ask(registration.active ??
|
|
49
|
-
view: async (path = location.pathname) => (await ask(registration.active ??
|
|
50
|
-
reportLoads: (id, count, origins) => (registration.active ??
|
|
89
|
+
status: () => ask(registration.active ?? active, { type: 'spool:status' }, timeout),
|
|
90
|
+
view: async (path = location.pathname) => (await ask(registration.active ?? active, { type: 'spool:view', path }, timeout)).id,
|
|
91
|
+
reportLoads: (id, count, origins) => (registration.active ?? active).postMessage({
|
|
51
92
|
type: 'spool:loads',
|
|
52
93
|
view: id,
|
|
53
94
|
count,
|
|
54
95
|
origins
|
|
55
96
|
}),
|
|
56
|
-
|
|
97
|
+
persisted: () => granted,
|
|
98
|
+
track: (db) => closeOnPageHide(db),
|
|
99
|
+
write: (name, key, value, version) => askWrite({ type: 'spool:write', name, key, value, version }),
|
|
100
|
+
flush: () => askWrite({ type: 'spool:flush' }, Math.max(timeout, FLUSH_WAIT_MS)),
|
|
101
|
+
writes: () => askWrite({ type: 'spool:writes' }),
|
|
102
|
+
resolve: (name, key, use) => askWrite({ type: 'spool:resolve', name, key, use }, Math.max(timeout, FLUSH_WAIT_MS))
|
|
57
103
|
};
|
|
58
104
|
}
|
|
@@ -3,13 +3,16 @@ import type { CollectorConfig } from './collector.js';
|
|
|
3
3
|
import type { SerializedRoute } from './route.js';
|
|
4
4
|
import type { CompiledTemplate } from './template/compile.js';
|
|
5
5
|
import type { Mode } from './view.js';
|
|
6
|
+
import type { WriteConfig } from './write.js';
|
|
6
7
|
/** every path under this prefix belongs to spool */
|
|
7
8
|
export declare const SPOOL_PREFIX = "/.spool/";
|
|
8
|
-
/** where the server answers sync, artifact and beacon requests, and where the worker script lives */
|
|
9
|
+
/** where the server answers sync, artifact, dataset and beacon requests, and where the worker script lives */
|
|
9
10
|
export declare const PATHS: {
|
|
10
11
|
readonly sync: "/.spool/sync";
|
|
11
12
|
readonly artifact: "/.spool/artifact";
|
|
13
|
+
readonly dataset: "/.spool/dataset";
|
|
12
14
|
readonly beacon: "/.spool/beacon";
|
|
15
|
+
readonly write: "/.spool/write";
|
|
13
16
|
readonly worker: "/spool-sw.js";
|
|
14
17
|
};
|
|
15
18
|
/** the same-origin path of the page runtime script for a given version */
|
|
@@ -86,4 +89,16 @@ export type SwConfig = {
|
|
|
86
89
|
collectors?: CollectorConfig[];
|
|
87
90
|
/** the opt-in cache for versioned cross-origin files */
|
|
88
91
|
thirdParty?: ThirdPartyConfig;
|
|
92
|
+
/**
|
|
93
|
+
* The share, above 0 and at most 1, of the storage headroom one dataset may use. The worker
|
|
94
|
+
* refuses a seed whose rows would use more; 1 when absent.
|
|
95
|
+
*/
|
|
96
|
+
datasetShare?: number;
|
|
97
|
+
/** the declared write classes; absent while declared writes are off, which is the default */
|
|
98
|
+
writes?: WriteConfig[];
|
|
99
|
+
/** the principal datasets and how long each may answer after its last confirmed sync; absent while the flag is off */
|
|
100
|
+
principalDatasets?: Array<{
|
|
101
|
+
name: string;
|
|
102
|
+
maxStaleMs: number;
|
|
103
|
+
}>;
|
|
89
104
|
};
|
package/dist/protocol/config.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
/** every path under this prefix belongs to spool */
|
|
2
2
|
export const SPOOL_PREFIX = '/.spool/';
|
|
3
|
-
/** where the server answers sync, artifact and beacon requests, and where the worker script lives */
|
|
3
|
+
/** where the server answers sync, artifact, dataset and beacon requests, and where the worker script lives */
|
|
4
4
|
export const PATHS = {
|
|
5
5
|
sync: '/.spool/sync',
|
|
6
6
|
artifact: '/.spool/artifact',
|
|
7
|
+
dataset: '/.spool/dataset',
|
|
7
8
|
beacon: '/.spool/beacon',
|
|
9
|
+
write: '/.spool/write',
|
|
8
10
|
worker: '/spool-sw.js'
|
|
9
11
|
};
|
|
10
12
|
/** the same-origin path of the page runtime script for a given version */
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { PROTOCOL_VERSION } from './artifact.js';
|
|
2
|
+
import type { Plain, SetProof } from './query.js';
|
|
3
|
+
import type { Identity } from './route.js';
|
|
4
|
+
/** a row of a dataset: a flat or nested JSON object, replaced whole when it changes */
|
|
5
|
+
export type DatasetRow = Plain;
|
|
6
|
+
/** what a dataset's functions receive; a public dataset has no identity */
|
|
7
|
+
export type DatasetContext = {
|
|
8
|
+
/** the request being answered */
|
|
9
|
+
request: Request;
|
|
10
|
+
/** the request URL, parsed */
|
|
11
|
+
url: URL;
|
|
12
|
+
/** who is asking; set for a principal dataset and absent, not null, for a public one */
|
|
13
|
+
principal?: Identity;
|
|
14
|
+
};
|
|
15
|
+
/** what `seed` receives: the context, and the key after which to start when the request pages */
|
|
16
|
+
export type DatasetSeedContext = DatasetContext & {
|
|
17
|
+
/** rows with a key at or before this are not wanted; a backend may use it to skip them */
|
|
18
|
+
after?: string;
|
|
19
|
+
};
|
|
20
|
+
/** a row that left the dataset, and the cursor at which it did */
|
|
21
|
+
export type DatasetDelete = {
|
|
22
|
+
key: string;
|
|
23
|
+
cursor: number;
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* What `changes` returns: the net change since a cursor. A key appears in `puts` or in `deletes`,
|
|
27
|
+
* never both. A row that entered the dataset's scope with its own version unchanged is a put; a row
|
|
28
|
+
* that left the scope is a delete. Deciding scope membership is `changes`' job, and spool cannot see
|
|
29
|
+
* it. `proof` is the set proof of the whole collection at `cursor`, when the backend keeps one.
|
|
30
|
+
*/
|
|
31
|
+
export type DatasetChanges = {
|
|
32
|
+
/** the cursor these changes bring a device to */
|
|
33
|
+
cursor: number;
|
|
34
|
+
/** keys of the rows to send whole; spool reads them with `read` */
|
|
35
|
+
puts: readonly string[];
|
|
36
|
+
/** keys of the rows to remove */
|
|
37
|
+
deletes: readonly DatasetDelete[];
|
|
38
|
+
/** the collection's proof at `cursor`; without it spool reads the whole collection to make one */
|
|
39
|
+
proof?: SetProof;
|
|
40
|
+
};
|
|
41
|
+
/** what `head` returns: the dataset's current cursor and, when the backend keeps one, its proof */
|
|
42
|
+
export type DatasetHead = {
|
|
43
|
+
/** the current cursor; a device that seeds at it asks for changes after it */
|
|
44
|
+
cursor: number;
|
|
45
|
+
/** the proof at `cursor` */
|
|
46
|
+
proof?: SetProof;
|
|
47
|
+
};
|
|
48
|
+
type MaybePromise<T> = T | Promise<T>;
|
|
49
|
+
/** the declaration `defineDataset` takes */
|
|
50
|
+
export type DatasetDeclaration<Row extends DatasetRow = DatasetRow> = {
|
|
51
|
+
/**
|
|
52
|
+
* `public` rows are the same for everyone. A `principal` dataset holds rows that differ per
|
|
53
|
+
* signed-in principal; it needs `maxStaleMs`, `defineDataset`'s `principalDatasets` option and
|
|
54
|
+
* `createSpool`'s `principalDatasets` flag. Unreviewed.
|
|
55
|
+
*/
|
|
56
|
+
scope: 'public' | 'principal';
|
|
57
|
+
/**
|
|
58
|
+
* Principal datasets only, and required there: how long held rows may answer a query after the
|
|
59
|
+
* last sync that returned and verified. Past it the query goes to the network. It is also the
|
|
60
|
+
* longest a device keeps serving rows after the server stopped admitting its principal.
|
|
61
|
+
*/
|
|
62
|
+
maxStaleMs?: number;
|
|
63
|
+
/** the row's key; unique, and stable while the row exists */
|
|
64
|
+
key(row: Row): string;
|
|
65
|
+
/**
|
|
66
|
+
* Every row in ascending key order (code unit order), as an array or as pages of rows. With
|
|
67
|
+
* `ctx.after` set it may start after that key. It runs once to admit a seed and once per page.
|
|
68
|
+
*/
|
|
69
|
+
seed(ctx: DatasetSeedContext): MaybePromise<readonly Row[] | Iterable<readonly Row[]> | AsyncIterable<readonly Row[]>>;
|
|
70
|
+
/** a keyed read: the rows for these keys, omitting any that no longer exist; never a full load */
|
|
71
|
+
read(ids: readonly string[], ctx: DatasetContext): MaybePromise<readonly Row[]>;
|
|
72
|
+
/** the net change since `since`, or `reset` when it cannot answer */
|
|
73
|
+
changes(since: number, ctx: DatasetContext): MaybePromise<DatasetChanges | 'reset'>;
|
|
74
|
+
/** the oldest cursor `changes` can still answer; a device older than this reseeds */
|
|
75
|
+
horizon(ctx: DatasetContext): MaybePromise<number>;
|
|
76
|
+
/** the current cursor, read before a seed so every later change reaches the device */
|
|
77
|
+
head(ctx: DatasetContext): MaybePromise<DatasetHead>;
|
|
78
|
+
/** a version string for the row shape; a device holding another one reseeds */
|
|
79
|
+
schema: string;
|
|
80
|
+
/** most rows a seed may hold */
|
|
81
|
+
maxRows: number;
|
|
82
|
+
/** most bytes of row JSON a seed may hold */
|
|
83
|
+
maxBytes: number;
|
|
84
|
+
/**
|
|
85
|
+
* `always` (the default) answers every seed. `auto` answers a seed only while `datasetBenefit`
|
|
86
|
+
* says the dataset beats both v0.3 and native HTTP; otherwise the seed is refused with `409`.
|
|
87
|
+
*/
|
|
88
|
+
provision?: 'always' | 'auto';
|
|
89
|
+
};
|
|
90
|
+
/** a validated dataset */
|
|
91
|
+
export type Dataset<Row extends DatasetRow = DatasetRow> = DatasetDeclaration<Row> & {
|
|
92
|
+
/** the name a device asks for */
|
|
93
|
+
name: string;
|
|
94
|
+
};
|
|
95
|
+
/** what `defineDataset` may be told besides the declaration */
|
|
96
|
+
export type DefineDatasetOptions = {
|
|
97
|
+
/** accept `scope: 'principal'`; `createSpool` still refuses it unless its own flag is on */
|
|
98
|
+
principalDatasets?: boolean;
|
|
99
|
+
};
|
|
100
|
+
/**
|
|
101
|
+
* Declares a dataset. Validates the name, the scope and the limits and returns the declaration with
|
|
102
|
+
* the name attached. Throws `bad-dataset` for a principal scope unless `options.principalDatasets`
|
|
103
|
+
* is true, and for a principal dataset without a positive `maxStaleMs` or with `provision: 'auto'`.
|
|
104
|
+
*/
|
|
105
|
+
export declare function defineDataset<Row extends DatasetRow = DatasetRow>(name: string, declaration: DatasetDeclaration<Row>, options?: DefineDatasetOptions): Dataset<Row>;
|
|
106
|
+
/** rows in a seed page when the request names no `limit` */
|
|
107
|
+
export declare const DATASET_PAGE_ROWS = 500;
|
|
108
|
+
/** the most rows one seed page may carry, whatever `limit` asks for */
|
|
109
|
+
export declare const DATASET_PAGE_CAP = 5000;
|
|
110
|
+
/** a row with its key, as it travels */
|
|
111
|
+
export type DatasetPut = {
|
|
112
|
+
key: string;
|
|
113
|
+
row: DatasetRow;
|
|
114
|
+
};
|
|
115
|
+
/**
|
|
116
|
+
* The body of `/.spool/dataset`. A patch has `reset: false` and `proof`. A seed, or a reset, has
|
|
117
|
+
* `reset: true` and one page of rows in `puts`: a page with `next` set is followed by a request
|
|
118
|
+
* with `after=<next>`, and the last page has no `next` and carries `proof`. A page after the first
|
|
119
|
+
* echoes that `after`.
|
|
120
|
+
*/
|
|
121
|
+
export type DatasetResponse = {
|
|
122
|
+
protocol: typeof PROTOCOL_VERSION;
|
|
123
|
+
name: string;
|
|
124
|
+
schema: string;
|
|
125
|
+
/** the cursor the response brings a device to */
|
|
126
|
+
cursor: number;
|
|
127
|
+
/** the device drops every row it holds before applying `puts` (unless `after` is set) */
|
|
128
|
+
reset: boolean;
|
|
129
|
+
puts: DatasetPut[];
|
|
130
|
+
deletes: DatasetDelete[];
|
|
131
|
+
/** on a seed page after the first: the key the request paged from */
|
|
132
|
+
after?: string;
|
|
133
|
+
/** on a seed page with more to come: the `after` of the next request */
|
|
134
|
+
next?: string;
|
|
135
|
+
/** the collection's proof at `cursor`; absent on a seed page that has a `next` */
|
|
136
|
+
proof?: SetProof;
|
|
137
|
+
};
|
|
138
|
+
/** what a device asks of `/.spool/dataset` */
|
|
139
|
+
export type DatasetRequest = {
|
|
140
|
+
name: string;
|
|
141
|
+
/** the cursor the device holds; absent for a seed */
|
|
142
|
+
since?: number;
|
|
143
|
+
/** the schema the device holds; required with `since` */
|
|
144
|
+
schema?: string;
|
|
145
|
+
/** the key to page from; seed pages only */
|
|
146
|
+
after?: string;
|
|
147
|
+
/** rows per seed page; the server's default when absent */
|
|
148
|
+
limit?: number;
|
|
149
|
+
};
|
|
150
|
+
/** the query string of a request, for `/.spool/dataset?` */
|
|
151
|
+
export declare function encodeDatasetRequest(request: DatasetRequest): string;
|
|
152
|
+
/** reads a request's parameters; throws `bad-dataset` for a missing name, a bad number, or `since` with `after` */
|
|
153
|
+
export declare function parseDatasetRequest(params: URLSearchParams): DatasetRequest;
|
|
154
|
+
/** validates JSON from `/.spool/dataset`; throws `bad-dataset` */
|
|
155
|
+
export declare function parseDatasetResponse(value: unknown): DatasetResponse;
|
|
156
|
+
/** what {@link applyChanges} returns */
|
|
157
|
+
export type Applied = {
|
|
158
|
+
/** the rows after the response, a new map */
|
|
159
|
+
rows: Map<string, DatasetRow>;
|
|
160
|
+
/** the collection's digest after the response */
|
|
161
|
+
digest: string;
|
|
162
|
+
/** whether the result matches `response.proof`; null when the response carries none */
|
|
163
|
+
matches: boolean | null;
|
|
164
|
+
/** the row hashes after the response; present when the caller passed `options.hashes` */
|
|
165
|
+
hashes?: Map<string, string>;
|
|
166
|
+
};
|
|
167
|
+
/** what {@link applyChanges} may be given besides the rows and the response */
|
|
168
|
+
export type ApplyOptions = {
|
|
169
|
+
/** the hash of every row in `rows`; a replaced or deleted row then costs no hash of its old copy */
|
|
170
|
+
hashes?: ReadonlyMap<string, string>;
|
|
171
|
+
/** the row hash function; {@link rowHash} when absent */
|
|
172
|
+
hash?: (key: string, row: DatasetRow) => Promise<string>;
|
|
173
|
+
};
|
|
174
|
+
/**
|
|
175
|
+
* Applies a response to `rows` and returns the result, leaving `rows` as it was. A reset response
|
|
176
|
+
* without `after` starts from no rows; a later seed page adds to what is held. Deletes apply
|
|
177
|
+
* before puts. `digest` is the digest of `rows` when the caller holds it, which makes the work one
|
|
178
|
+
* row hash per replaced, deleted or added row; without it the digest of `rows` is computed first.
|
|
179
|
+
* With `options.hashes` a replaced or deleted row's old hash is read, not computed, and the result
|
|
180
|
+
* carries the hashes after the response. Pure: no I/O, no clock.
|
|
181
|
+
*/
|
|
182
|
+
export declare function applyChanges(rows: ReadonlyMap<string, DatasetRow>, response: DatasetResponse, digest?: string, options?: ApplyOptions): Promise<Applied>;
|
|
183
|
+
export {};
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import { PROTOCOL_VERSION } from './artifact.js';
|
|
2
|
+
import { ProtocolError } from './errors.js';
|
|
3
|
+
import { DATASET_NAME, digestAdd, digestSub, EMPTY_DIGEST, isSetProof, rowHash, setProof } from './query.js';
|
|
4
|
+
const NAME = DATASET_NAME;
|
|
5
|
+
function bad(name, message) {
|
|
6
|
+
throw new ProtocolError('bad-dataset', `${name}: ${message}`);
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Declares a dataset. Validates the name, the scope and the limits and returns the declaration with
|
|
10
|
+
* the name attached. Throws `bad-dataset` for a principal scope unless `options.principalDatasets`
|
|
11
|
+
* is true, and for a principal dataset without a positive `maxStaleMs` or with `provision: 'auto'`.
|
|
12
|
+
*/
|
|
13
|
+
export function defineDataset(name, declaration, options = {}) {
|
|
14
|
+
if (typeof name !== 'string' || !NAME.test(name)) {
|
|
15
|
+
return bad(String(name), 'a dataset name is 1 to 128 of letters, digits and . _ : -');
|
|
16
|
+
}
|
|
17
|
+
const scope = declaration.scope;
|
|
18
|
+
if (scope === 'principal' && options.principalDatasets !== true) {
|
|
19
|
+
return bad(name, 'a principal dataset needs the principal datasets flag: pass { principalDatasets: true } here and set the principalDatasets option of createSpool');
|
|
20
|
+
}
|
|
21
|
+
if (scope !== 'public' && scope !== 'principal')
|
|
22
|
+
return bad(name, `unknown scope "${scope}"`);
|
|
23
|
+
if (scope === 'principal') {
|
|
24
|
+
const stale = declaration.maxStaleMs;
|
|
25
|
+
if (typeof stale !== 'number' || !Number.isSafeInteger(stale) || stale < 1) {
|
|
26
|
+
return bad(name, 'a principal dataset needs maxStaleMs, a positive integer of milliseconds');
|
|
27
|
+
}
|
|
28
|
+
if (declaration.provision === 'auto') {
|
|
29
|
+
return bad(name, 'a principal dataset cannot use provision "auto"');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
else if (declaration.maxStaleMs !== undefined) {
|
|
33
|
+
return bad(name, 'maxStaleMs applies to a principal dataset');
|
|
34
|
+
}
|
|
35
|
+
for (const fn of ['key', 'seed', 'read', 'changes', 'horizon', 'head']) {
|
|
36
|
+
if (typeof declaration[fn] !== 'function')
|
|
37
|
+
return bad(name, `${fn} must be a function`);
|
|
38
|
+
}
|
|
39
|
+
if (typeof declaration.schema !== 'string' || declaration.schema === '') {
|
|
40
|
+
return bad(name, 'schema must be a non-empty string');
|
|
41
|
+
}
|
|
42
|
+
for (const limit of ['maxRows', 'maxBytes']) {
|
|
43
|
+
const value = declaration[limit];
|
|
44
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1) {
|
|
45
|
+
return bad(name, `${limit} must be a positive integer`);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (declaration.provision !== undefined &&
|
|
49
|
+
!['always', 'auto'].includes(declaration.provision)) {
|
|
50
|
+
return bad(name, 'provision must be "always" or "auto"');
|
|
51
|
+
}
|
|
52
|
+
return { ...declaration, name };
|
|
53
|
+
}
|
|
54
|
+
/** rows in a seed page when the request names no `limit` */
|
|
55
|
+
export const DATASET_PAGE_ROWS = 500;
|
|
56
|
+
/** the most rows one seed page may carry, whatever `limit` asks for */
|
|
57
|
+
export const DATASET_PAGE_CAP = 5000;
|
|
58
|
+
function reject(message) {
|
|
59
|
+
throw new ProtocolError('bad-dataset', message);
|
|
60
|
+
}
|
|
61
|
+
const INTEGER = /^\d{1,15}$/;
|
|
62
|
+
/** the query string of a request, for `/.spool/dataset?` */
|
|
63
|
+
export function encodeDatasetRequest(request) {
|
|
64
|
+
const params = new URLSearchParams({ name: request.name });
|
|
65
|
+
if (request.since !== undefined)
|
|
66
|
+
params.set('since', String(request.since));
|
|
67
|
+
if (request.schema !== undefined)
|
|
68
|
+
params.set('schema', request.schema);
|
|
69
|
+
if (request.after !== undefined)
|
|
70
|
+
params.set('after', request.after);
|
|
71
|
+
if (request.limit !== undefined)
|
|
72
|
+
params.set('limit', String(request.limit));
|
|
73
|
+
return params.toString();
|
|
74
|
+
}
|
|
75
|
+
/** reads a request's parameters; throws `bad-dataset` for a missing name, a bad number, or `since` with `after` */
|
|
76
|
+
export function parseDatasetRequest(params) {
|
|
77
|
+
const name = params.get('name');
|
|
78
|
+
if (name === null || !NAME.test(name))
|
|
79
|
+
return reject('name is required');
|
|
80
|
+
const out = { name };
|
|
81
|
+
const since = params.get('since');
|
|
82
|
+
if (since !== null) {
|
|
83
|
+
if (!INTEGER.test(since))
|
|
84
|
+
return reject('since must be a non-negative integer');
|
|
85
|
+
out.since = Number(since);
|
|
86
|
+
}
|
|
87
|
+
const schema = params.get('schema');
|
|
88
|
+
if (schema !== null)
|
|
89
|
+
out.schema = schema;
|
|
90
|
+
const after = params.get('after');
|
|
91
|
+
if (after !== null)
|
|
92
|
+
out.after = after;
|
|
93
|
+
const limit = params.get('limit');
|
|
94
|
+
if (limit !== null) {
|
|
95
|
+
if (!/^[1-9]\d{0,8}$/.test(limit))
|
|
96
|
+
return reject('limit must be a positive integer');
|
|
97
|
+
out.limit = Math.min(Number(limit), DATASET_PAGE_CAP);
|
|
98
|
+
}
|
|
99
|
+
if (out.since !== undefined && out.schema === undefined) {
|
|
100
|
+
return reject('since needs the schema the device holds');
|
|
101
|
+
}
|
|
102
|
+
if (out.since !== undefined && out.after !== undefined) {
|
|
103
|
+
return reject('since and after are exclusive');
|
|
104
|
+
}
|
|
105
|
+
return out;
|
|
106
|
+
}
|
|
107
|
+
const isPlain = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
108
|
+
const isCursor = (value) => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
|
|
109
|
+
/** validates JSON from `/.spool/dataset`; throws `bad-dataset` */
|
|
110
|
+
export function parseDatasetResponse(value) {
|
|
111
|
+
if (!isPlain(value))
|
|
112
|
+
return reject('dataset response must be an object');
|
|
113
|
+
if (value['protocol'] !== PROTOCOL_VERSION)
|
|
114
|
+
return reject('unsupported protocol version');
|
|
115
|
+
const { name, schema, cursor, reset, puts, deletes, after, next, proof } = value;
|
|
116
|
+
if (typeof name !== 'string' || typeof schema !== 'string') {
|
|
117
|
+
return reject('name and schema must be strings');
|
|
118
|
+
}
|
|
119
|
+
if (!isCursor(cursor))
|
|
120
|
+
return reject('cursor must be a non-negative integer');
|
|
121
|
+
if (typeof reset !== 'boolean')
|
|
122
|
+
return reject('reset must be a boolean');
|
|
123
|
+
if (!Array.isArray(puts))
|
|
124
|
+
return reject('puts must be a list');
|
|
125
|
+
const outPuts = puts.map((put) => {
|
|
126
|
+
if (!isPlain(put) || typeof put['key'] !== 'string' || !isPlain(put['row'])) {
|
|
127
|
+
return reject('a put is { key, row }');
|
|
128
|
+
}
|
|
129
|
+
return { key: put['key'], row: put['row'] };
|
|
130
|
+
});
|
|
131
|
+
if (!Array.isArray(deletes))
|
|
132
|
+
return reject('deletes must be a list');
|
|
133
|
+
const outDeletes = deletes.map((del) => {
|
|
134
|
+
if (!isPlain(del) || typeof del['key'] !== 'string' || !isCursor(del['cursor'])) {
|
|
135
|
+
return reject('a delete is { key, cursor }');
|
|
136
|
+
}
|
|
137
|
+
return { key: del['key'], cursor: del['cursor'] };
|
|
138
|
+
});
|
|
139
|
+
const out = {
|
|
140
|
+
protocol: PROTOCOL_VERSION,
|
|
141
|
+
name,
|
|
142
|
+
schema,
|
|
143
|
+
cursor,
|
|
144
|
+
reset,
|
|
145
|
+
puts: outPuts,
|
|
146
|
+
deletes: outDeletes
|
|
147
|
+
};
|
|
148
|
+
if (after !== undefined) {
|
|
149
|
+
if (typeof after !== 'string')
|
|
150
|
+
return reject('after must be a string');
|
|
151
|
+
out.after = after;
|
|
152
|
+
}
|
|
153
|
+
if (next !== undefined) {
|
|
154
|
+
if (typeof next !== 'string')
|
|
155
|
+
return reject('next must be a string');
|
|
156
|
+
out.next = next;
|
|
157
|
+
}
|
|
158
|
+
if (proof !== undefined) {
|
|
159
|
+
if (!isSetProof(proof))
|
|
160
|
+
return reject('proof must be a set proof');
|
|
161
|
+
out.proof = { kind: 'set', count: proof.count, digest: proof.digest };
|
|
162
|
+
}
|
|
163
|
+
if (out.next === undefined && out.proof === undefined) {
|
|
164
|
+
return reject('a response without next carries a proof');
|
|
165
|
+
}
|
|
166
|
+
return out;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Applies a response to `rows` and returns the result, leaving `rows` as it was. A reset response
|
|
170
|
+
* without `after` starts from no rows; a later seed page adds to what is held. Deletes apply
|
|
171
|
+
* before puts. `digest` is the digest of `rows` when the caller holds it, which makes the work one
|
|
172
|
+
* row hash per replaced, deleted or added row; without it the digest of `rows` is computed first.
|
|
173
|
+
* With `options.hashes` a replaced or deleted row's old hash is read, not computed, and the result
|
|
174
|
+
* carries the hashes after the response. Pure: no I/O, no clock.
|
|
175
|
+
*/
|
|
176
|
+
export async function applyChanges(rows, response, digest, options = {}) {
|
|
177
|
+
const hash = options.hash ?? rowHash;
|
|
178
|
+
const fresh = response.reset && response.after === undefined;
|
|
179
|
+
const out = new Map(fresh ? [] : rows);
|
|
180
|
+
const hashes = options.hashes ? new Map(fresh ? [] : options.hashes) : undefined;
|
|
181
|
+
let sum = fresh ? EMPTY_DIGEST : (digest ?? (await setProof(rows)).digest);
|
|
182
|
+
const drop = async (key) => {
|
|
183
|
+
const old = out.get(key);
|
|
184
|
+
if (old === undefined)
|
|
185
|
+
return;
|
|
186
|
+
sum = digestSub(sum, hashes?.get(key) ?? (await hash(key, old)));
|
|
187
|
+
out.delete(key);
|
|
188
|
+
hashes?.delete(key);
|
|
189
|
+
};
|
|
190
|
+
for (const { key } of response.deletes)
|
|
191
|
+
await drop(key);
|
|
192
|
+
for (const { key, row } of response.puts) {
|
|
193
|
+
await drop(key);
|
|
194
|
+
out.set(key, row);
|
|
195
|
+
const added = await hash(key, row);
|
|
196
|
+
hashes?.set(key, added);
|
|
197
|
+
sum = digestAdd(sum, added);
|
|
198
|
+
}
|
|
199
|
+
const proof = response.proof;
|
|
200
|
+
return {
|
|
201
|
+
rows: out,
|
|
202
|
+
digest: sum,
|
|
203
|
+
matches: proof === undefined ? null : proof.count === out.size && proof.digest === sum,
|
|
204
|
+
...(hashes ? { hashes } : {})
|
|
205
|
+
};
|
|
206
|
+
}
|
|
@@ -9,6 +9,7 @@ export declare const HEADERS: {
|
|
|
9
9
|
readonly partition: "spool-partition";
|
|
10
10
|
readonly vectorSeen: "spool-vector-seen";
|
|
11
11
|
readonly provision: "spool-provision";
|
|
12
|
+
readonly provisionReason: "spool-provision-reason";
|
|
12
13
|
};
|
|
13
14
|
/** `Spool-Clear` token that wipes everything the worker holds */
|
|
14
15
|
export declare const CLEAR_ALL = "*";
|
|
@@ -9,7 +9,8 @@ export const HEADERS = {
|
|
|
9
9
|
vector: 'spool-vector',
|
|
10
10
|
partition: 'spool-partition',
|
|
11
11
|
vectorSeen: 'spool-vector-seen',
|
|
12
|
-
provision: 'spool-provision'
|
|
12
|
+
provision: 'spool-provision',
|
|
13
|
+
provisionReason: 'spool-provision-reason'
|
|
13
14
|
};
|
|
14
15
|
/** `Spool-Clear` token that wipes everything the worker holds */
|
|
15
16
|
export const CLEAR_ALL = '*';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** why a protocol value was refused */
|
|
2
|
-
export type ProtocolErrorCode = 'bad-pattern' | 'bad-route' | 'bad-header' | 'bad-artifact' | 'bad-sync' | 'bad-beacon' | 'bad-template' | 'bad-manifest' | 'bad-nav';
|
|
2
|
+
export type ProtocolErrorCode = 'bad-pattern' | 'bad-route' | 'bad-header' | 'bad-artifact' | 'bad-sync' | 'bad-beacon' | 'bad-template' | 'bad-manifest' | 'bad-nav' | 'bad-dataset' | 'dataset-limit' | 'dataset-forbidden' | 'bad-write';
|
|
3
3
|
/** thrown for malformed protocol input; `code` names the family so callers can map it to a status */
|
|
4
4
|
export declare class ProtocolError extends Error {
|
|
5
5
|
readonly code: ProtocolErrorCode;
|
package/dist/protocol/graph.d.ts
CHANGED
|
@@ -6,8 +6,15 @@ export type RecordRef = {
|
|
|
6
6
|
export type ProjectionRef = {
|
|
7
7
|
$projection: string;
|
|
8
8
|
};
|
|
9
|
+
/**
|
|
10
|
+
* a reference to a row of a dataset the device holds, as `<dataset>/<key>`; the page keeps the id
|
|
11
|
+
* and the row resolves when the page renders, so a change to the row needs no new artifact
|
|
12
|
+
*/
|
|
13
|
+
export type RowRef = {
|
|
14
|
+
$row: string;
|
|
15
|
+
};
|
|
9
16
|
/** what a page's data carries in place of a record or projection stored once elsewhere */
|
|
10
|
-
export type Ref = RecordRef | ProjectionRef;
|
|
17
|
+
export type Ref = RecordRef | ProjectionRef | RowRef;
|
|
11
18
|
/** an authoritative piece of data, stored once by id; its id is the tag that invalidates it */
|
|
12
19
|
export type RecordEntry = {
|
|
13
20
|
id: string;
|
|
@@ -24,23 +31,33 @@ export type ProjectionEntry = {
|
|
|
24
31
|
/** a tree that may reference records and other projections */
|
|
25
32
|
data: unknown;
|
|
26
33
|
};
|
|
27
|
-
/** the id lists {@link collectRefs} found */
|
|
34
|
+
/** the id lists {@link collectRefs} found; `rows` is set only when there are row references */
|
|
28
35
|
export type RefSet = {
|
|
29
36
|
records: string[];
|
|
30
37
|
projections: string[];
|
|
38
|
+
rows?: string[];
|
|
31
39
|
};
|
|
32
40
|
/** the reference to put in a page's data for record `id` */
|
|
33
41
|
export declare function recordRef(id: string): RecordRef;
|
|
34
42
|
/** the reference to put in a page's data for projection `id` */
|
|
35
43
|
export declare function projectionRef(id: string): ProjectionRef;
|
|
44
|
+
/** the reference to put in a page's data for the row `key` of dataset `dataset` */
|
|
45
|
+
export declare function rowRef(dataset: string, key: string): RowRef;
|
|
46
|
+
/** splits a row id into its dataset name and key; the name never holds a `/`, so the first one splits */
|
|
47
|
+
export declare function splitRowId(id: string): {
|
|
48
|
+
dataset: string;
|
|
49
|
+
key: string;
|
|
50
|
+
} | null;
|
|
36
51
|
/** whether `value` is exactly a reference, with no other keys */
|
|
37
52
|
export declare function isRef(value: unknown): value is Ref;
|
|
38
|
-
/** every record and
|
|
53
|
+
/** every record, projection and row id referenced anywhere in `value`, sorted and unique */
|
|
39
54
|
export declare function collectRefs(value: unknown): RefSet;
|
|
40
|
-
/** where {@link resolveRefs} looks references up */
|
|
55
|
+
/** where {@link resolveRefs} looks references up; without `row`, a row reference is a refusal */
|
|
41
56
|
export type RefLookup = {
|
|
42
57
|
record(id: string): RecordEntry | undefined;
|
|
43
58
|
projection(id: string): ProjectionEntry | undefined;
|
|
59
|
+
/** the held row `key` of `dataset`, or undefined when it is not held */
|
|
60
|
+
row?(dataset: string, key: string): unknown;
|
|
44
61
|
};
|
|
45
62
|
/** the tree with every reference replaced by what it points at, or the first one that is missing */
|
|
46
63
|
export type Resolved = {
|
|
@@ -71,6 +88,8 @@ export type Collected = {
|
|
|
71
88
|
export type Collector = {
|
|
72
89
|
record(id: string, data: unknown): RecordRef;
|
|
73
90
|
projection(id: string, data: unknown, deps?: readonly string[]): ProjectionRef;
|
|
91
|
+
/** the reference to a dataset row; nothing is registered, the device resolves it from its rows */
|
|
92
|
+
row(dataset: string, key: string): RowRef;
|
|
74
93
|
/** everything registered so far; throws when a projection reaches an unregistered or cyclic one */
|
|
75
94
|
collect(version: number): Collected;
|
|
76
95
|
};
|