@drupflare/spool 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +499 -43
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/activity.d.ts +37 -0
- package/dist/client/activity.js +114 -0
- package/dist/client/entry.js +34 -1
- package/dist/client/index.d.ts +1 -0
- package/dist/client/index.js +1 -0
- package/dist/client/register.d.ts +23 -1
- package/dist/client/register.js +53 -7
- package/dist/media/embeds.d.ts +3 -2
- package/dist/media/embeds.js +16 -1
- package/dist/protocol/config.d.ts +16 -1
- package/dist/protocol/config.js +3 -1
- package/dist/protocol/dataset.d.ts +183 -0
- package/dist/protocol/dataset.js +206 -0
- package/dist/protocol/envelope.d.ts +3 -0
- package/dist/protocol/envelope.js +14 -3
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/hash.d.ts +2 -0
- package/dist/protocol/hash.js +8 -0
- package/dist/protocol/index.d.ts +7 -0
- package/dist/protocol/index.js +7 -0
- package/dist/protocol/inline.d.ts +14 -0
- package/dist/protocol/inline.js +43 -0
- package/dist/protocol/navigation.d.ts +81 -0
- package/dist/protocol/navigation.js +249 -0
- package/dist/protocol/normalise.d.ts +4 -3
- package/dist/protocol/normalise.js +6 -4
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +52 -1
- package/dist/protocol/provision.js +53 -5
- package/dist/protocol/query.d.ts +141 -0
- package/dist/protocol/query.js +252 -0
- package/dist/protocol/route.d.ts +85 -5
- package/dist/protocol/route.js +91 -0
- package/dist/protocol/write.d.ts +146 -0
- package/dist/protocol/write.js +101 -0
- package/dist/proxy/bases.d.ts +53 -0
- package/dist/proxy/bases.js +76 -0
- package/dist/proxy/index.d.ts +1 -0
- package/dist/proxy/index.js +1 -0
- package/dist/proxy/proxy.js +8 -2
- package/dist/report/capacity.d.ts +59 -0
- package/dist/report/capacity.js +76 -0
- package/dist/report/channels.d.ts +60 -0
- package/dist/report/channels.js +65 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +11 -1
- package/dist/report/economics.js +19 -1
- package/dist/report/index.d.ts +5 -0
- package/dist/report/index.js +5 -0
- package/dist/report/inputs.d.ts +97 -0
- package/dist/report/inputs.js +133 -0
- package/dist/report/institution.d.ts +284 -0
- package/dist/report/institution.js +225 -0
- package/dist/report/ledger.d.ts +28 -0
- package/dist/report/ledger.js +30 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +6 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/traffic.d.ts +11 -2
- package/dist/report/traffic.js +13 -3
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/dataset.d.ts +48 -0
- package/dist/server/dataset.js +308 -0
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +3 -0
- package/dist/server/index.js +3 -0
- package/dist/server/spool.d.ts +40 -0
- package/dist/server/spool.js +184 -23
- package/dist/server/write.d.ts +27 -0
- package/dist/server/write.js +127 -0
- package/dist/sw/analytics.d.ts +6 -0
- package/dist/sw/analytics.js +14 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.js +78 -9
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/render.d.ts +8 -1
- package/dist/sw/render.js +28 -1
- package/dist/sw/replica.d.ts +14 -3
- package/dist/sw/replica.js +46 -5
- package/dist/sw/router.js +2 -2
- package/dist/sw/runtime.d.ts +43 -2
- package/dist/sw/runtime.js +440 -29
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +107 -0
- package/dist/sw/write.js +442 -0
- package/package.json +5 -2
|
@@ -78,6 +78,8 @@ export type EntryCandidate = {
|
|
|
78
78
|
path: string;
|
|
79
79
|
/** the predicted share of devices that will open the page, 0 to 1 */
|
|
80
80
|
p: number;
|
|
81
|
+
/** the observations behind `p`; with a `z` it lowers `p` to a confidence bound, so a share seen once does not count as one seen a thousand times */
|
|
82
|
+
weight?: number;
|
|
81
83
|
/** compressed bytes of the entry */
|
|
82
84
|
entryBytes: number;
|
|
83
85
|
/** bytes of the document the entry replaces */
|
|
@@ -87,10 +89,13 @@ export type EntryCandidate = {
|
|
|
87
89
|
* The candidates worth shipping: those whose `p` is at least their break-even, best expected net
|
|
88
90
|
* first, kept while their entry bytes fit under `capBytes`. A candidate that does not fit is
|
|
89
91
|
* skipped and a smaller one may still go. A path is chosen once. A candidate with a `p` outside 0
|
|
90
|
-
* to 1 or a negative or non-finite size is never chosen.
|
|
92
|
+
* to 1 or a negative or non-finite size is never chosen. With `z` above 0, a candidate that has a
|
|
93
|
+
* `weight` is judged on the lower confidence bound of its `p` (the Wilson bound at `z`), and the
|
|
94
|
+
* order follows the net at that bound; one with no `weight` keeps its `p`.
|
|
91
95
|
*/
|
|
92
96
|
export declare function chooseEntries(candidates: readonly EntryCandidate[], options: {
|
|
93
97
|
capBytes: number;
|
|
98
|
+
z?: number;
|
|
94
99
|
}): EntryCandidate[];
|
|
95
100
|
/**
|
|
96
101
|
* P_reuse from aggregate telemetry: the share of a route's views that found a stored copy, pulled
|
|
@@ -98,3 +103,49 @@ export declare function chooseEntries(candidates: readonly EntryCandidate[], opt
|
|
|
98
103
|
* has no device identity, so this is a share of views, not of provisioned copies.
|
|
99
104
|
*/
|
|
100
105
|
export declare function smoothReuse(warmViews: number, coldViews: number, prior?: ProvisionPrior, weight?: number): number;
|
|
106
|
+
/** the two controls a dataset must beat: the held page under v0.3, and a plain HTTP cache */
|
|
107
|
+
export type DatasetControl = 'v0.3' | 'native';
|
|
108
|
+
/** compressed sizes, in bytes, of one dataset and the two controls it is measured against */
|
|
109
|
+
export type DatasetInputs = {
|
|
110
|
+
/** the seed the device stores first */
|
|
111
|
+
seedBytes: number;
|
|
112
|
+
/** one update's patch */
|
|
113
|
+
patchBytes: number;
|
|
114
|
+
/** updates per day to the rows the dataset holds */
|
|
115
|
+
updatesPerDay: number;
|
|
116
|
+
/** v0.3: the seed of the held artifact and the bytes it refetches per update */
|
|
117
|
+
v03: {
|
|
118
|
+
seedBytes: number;
|
|
119
|
+
refetchBytes: number;
|
|
120
|
+
};
|
|
121
|
+
/** native: the bytes a plain HTTP cache still sends for each view */
|
|
122
|
+
nativeBytesPerView: number;
|
|
123
|
+
/** chance a stored copy survives the hold window; see {@link DATASET_DEFAULTS} */
|
|
124
|
+
survival?: number;
|
|
125
|
+
/** days a copy is held and kept current; see {@link DATASET_DEFAULTS} */
|
|
126
|
+
holdDays?: number;
|
|
127
|
+
};
|
|
128
|
+
/**
|
|
129
|
+
* Scenario values (L3). Safari clears script-written storage after 7 days without interaction
|
|
130
|
+
* (the window is published, L0); the survival of a copy inside that window is not measured.
|
|
131
|
+
*/
|
|
132
|
+
export declare const DATASET_DEFAULTS: {
|
|
133
|
+
readonly survival: 0.5;
|
|
134
|
+
readonly holdDays: 7;
|
|
135
|
+
};
|
|
136
|
+
export type DatasetBenefit = {
|
|
137
|
+
/** weighed bytes the dataset saves against each control; positive means it moves fewer */
|
|
138
|
+
net: Record<DatasetControl, number>;
|
|
139
|
+
/** the control with the smaller margin: the one the dataset must beat hardest */
|
|
140
|
+
controls: DatasetControl;
|
|
141
|
+
/** the dataset beats both controls */
|
|
142
|
+
ship: boolean;
|
|
143
|
+
};
|
|
144
|
+
/**
|
|
145
|
+
* Views a stored copy serves are `reuse x survival x futureViews`. A device fetches at most one
|
|
146
|
+
* patch per served view, so the fetches over the hold are `min(updates, views)`. Each control
|
|
147
|
+
* is compared on the same views: the margin against v0.3 is its seed and refetches less the
|
|
148
|
+
* dataset's; the margin against native is what a plain HTTP cache would still send for those
|
|
149
|
+
* views less the dataset's seed and patches. Requests weigh as {@link weigh} does.
|
|
150
|
+
*/
|
|
151
|
+
export declare function datasetBenefit(inputs: DatasetInputs, prior?: ProvisionPrior): DatasetBenefit;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { lowerBound } from './navigation.js';
|
|
1
2
|
/** the defaults a config without `provision` gets */
|
|
2
3
|
export const DEFAULT_PRIOR = {
|
|
3
4
|
reuse: 0.5,
|
|
@@ -65,24 +66,33 @@ export function entryBreakEven(entryBytes, documentBytes) {
|
|
|
65
66
|
return (JOULES_PER_BYTE * entryBytes) / (JOULES_PER_BYTE * documentBytes + ORIGIN_JOULES);
|
|
66
67
|
}
|
|
67
68
|
const sound = (n) => Number.isFinite(n) && n >= 0;
|
|
69
|
+
/** the share the decision uses: `p`, or its lower confidence bound at `z` when the candidate says how many observations back it */
|
|
70
|
+
function shareOf(c, z) {
|
|
71
|
+
return c.weight !== undefined && z > 0 ? lowerBound(c.p, c.weight, z) : c.p;
|
|
72
|
+
}
|
|
68
73
|
/** expected joules of shipping an entry: `p * (k*D + o) - k*E` */
|
|
69
|
-
function expectedNet(c) {
|
|
70
|
-
return (c
|
|
74
|
+
function expectedNet(c, z) {
|
|
75
|
+
return (shareOf(c, z) * (JOULES_PER_BYTE * c.documentBytes + ORIGIN_JOULES) -
|
|
76
|
+
JOULES_PER_BYTE * c.entryBytes);
|
|
71
77
|
}
|
|
72
78
|
/**
|
|
73
79
|
* The candidates worth shipping: those whose `p` is at least their break-even, best expected net
|
|
74
80
|
* first, kept while their entry bytes fit under `capBytes`. A candidate that does not fit is
|
|
75
81
|
* skipped and a smaller one may still go. A path is chosen once. A candidate with a `p` outside 0
|
|
76
|
-
* to 1 or a negative or non-finite size is never chosen.
|
|
82
|
+
* to 1 or a negative or non-finite size is never chosen. With `z` above 0, a candidate that has a
|
|
83
|
+
* `weight` is judged on the lower confidence bound of its `p` (the Wilson bound at `z`), and the
|
|
84
|
+
* order follows the net at that bound; one with no `weight` keeps its `p`.
|
|
77
85
|
*/
|
|
78
86
|
export function chooseEntries(candidates, options) {
|
|
87
|
+
const z = options.z ?? 0;
|
|
79
88
|
const paying = candidates
|
|
80
89
|
.filter((c) => sound(c.entryBytes) &&
|
|
81
90
|
sound(c.documentBytes) &&
|
|
82
91
|
c.p >= 0 &&
|
|
83
92
|
c.p <= 1 &&
|
|
84
|
-
c
|
|
85
|
-
.sort((a, b) => expectedNet(b) - expectedNet(a) ||
|
|
93
|
+
shareOf(c, z) >= entryBreakEven(c.entryBytes, c.documentBytes))
|
|
94
|
+
.sort((a, b) => expectedNet(b, z) - expectedNet(a, z) ||
|
|
95
|
+
(a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
|
|
86
96
|
const out = [];
|
|
87
97
|
const seen = new Set();
|
|
88
98
|
let bytes = 0;
|
|
@@ -103,3 +113,41 @@ export function chooseEntries(candidates, options) {
|
|
|
103
113
|
export function smoothReuse(warmViews, coldViews, prior = DEFAULT_PRIOR, weight = 10) {
|
|
104
114
|
return (prior.reuse * weight + warmViews) / (weight + warmViews + coldViews);
|
|
105
115
|
}
|
|
116
|
+
/**
|
|
117
|
+
* Scenario values (L3). Safari clears script-written storage after 7 days without interaction
|
|
118
|
+
* (the window is published, L0); the survival of a copy inside that window is not measured.
|
|
119
|
+
*/
|
|
120
|
+
export const DATASET_DEFAULTS = { survival: 0.5, holdDays: 7 };
|
|
121
|
+
/**
|
|
122
|
+
* Views a stored copy serves are `reuse x survival x futureViews`. A device fetches at most one
|
|
123
|
+
* patch per served view, so the fetches over the hold are `min(updates, views)`. Each control
|
|
124
|
+
* is compared on the same views: the margin against v0.3 is its seed and refetches less the
|
|
125
|
+
* dataset's; the margin against native is what a plain HTTP cache would still send for those
|
|
126
|
+
* views less the dataset's seed and patches. Requests weigh as {@link weigh} does.
|
|
127
|
+
*/
|
|
128
|
+
export function datasetBenefit(inputs, prior = DEFAULT_PRIOR) {
|
|
129
|
+
const survival = inputs.survival ?? DATASET_DEFAULTS.survival;
|
|
130
|
+
const holdDays = inputs.holdDays ?? DATASET_DEFAULTS.holdDays;
|
|
131
|
+
const sizes = [
|
|
132
|
+
inputs.seedBytes,
|
|
133
|
+
inputs.patchBytes,
|
|
134
|
+
inputs.updatesPerDay,
|
|
135
|
+
inputs.v03.seedBytes,
|
|
136
|
+
inputs.v03.refetchBytes,
|
|
137
|
+
inputs.nativeBytesPerView,
|
|
138
|
+
holdDays
|
|
139
|
+
];
|
|
140
|
+
if (!sizes.every(sound) || !(survival >= 0 && survival <= 1)) {
|
|
141
|
+
throw new RangeError('dataset sizes must be non-negative numbers and survival 0 to 1');
|
|
142
|
+
}
|
|
143
|
+
const w = (bytes) => weigh({ bytes, requests: 1 }, prior.requestBytes);
|
|
144
|
+
const views = prior.reuse * survival * prior.futureViews;
|
|
145
|
+
const fetches = Math.min(inputs.updatesPerDay * holdDays, views);
|
|
146
|
+
const dataset = w(inputs.seedBytes) + fetches * w(inputs.patchBytes);
|
|
147
|
+
const net = {
|
|
148
|
+
'v0.3': w(inputs.v03.seedBytes) + fetches * w(inputs.v03.refetchBytes) - dataset,
|
|
149
|
+
native: views * w(inputs.nativeBytesPerView) - dataset
|
|
150
|
+
};
|
|
151
|
+
const controls = net.native < net['v0.3'] ? 'native' : 'v0.3';
|
|
152
|
+
return { net, controls, ship: net[controls] > 0 };
|
|
153
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import type { QueryPlan } from './plan.js';
|
|
2
|
+
/** how a query parameter is read when it filters a field */
|
|
3
|
+
export type QueryFilterType = 'string' | 'number';
|
|
4
|
+
/**
|
|
5
|
+
* The query strings a public route answers from the collection it holds. `list` names the key of the
|
|
6
|
+
* loaded data that holds the whole collection, an array of flat objects. A parameter named in
|
|
7
|
+
* `filter` keeps the items whose field of the same name equals it, and a repeated one matches any of
|
|
8
|
+
* its values. `sort` lists the fields the `sort` parameter may name (`-name` sorts descending), and
|
|
9
|
+
* `pageSize` turns on the 1-based `page` parameter.
|
|
10
|
+
*/
|
|
11
|
+
export type QuerySchema = {
|
|
12
|
+
/** key of the loaded data that holds the collection */
|
|
13
|
+
list: string;
|
|
14
|
+
/** parameters that filter items, by field name, with how each is read */
|
|
15
|
+
filter?: Record<string, QueryFilterType>;
|
|
16
|
+
/** fields the sort parameter may name; a leading minus reverses the order */
|
|
17
|
+
sort?: readonly string[];
|
|
18
|
+
/** items per page; turns on the page parameter */
|
|
19
|
+
pageSize?: number;
|
|
20
|
+
/** field of each item that holds its dataset key; a list with a set proof needs it */
|
|
21
|
+
key?: string;
|
|
22
|
+
/**
|
|
23
|
+
* Name of a dataset the device holds. The rows it holds are the collection, in ascending key
|
|
24
|
+
* order, so the route's data carries no list and no proof. Exclusive with `key`.
|
|
25
|
+
*/
|
|
26
|
+
dataset?: string;
|
|
27
|
+
/**
|
|
28
|
+
* A read-only plan over the held dataset: joins, conditions, groups and aggregates. Requires
|
|
29
|
+
* `dataset`; `list` names the key the result rows go under. See {@link QueryPlan}.
|
|
30
|
+
*/
|
|
31
|
+
plan?: QueryPlan;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* What a collection's `load` result carries so the worker can tell it holds every item the server
|
|
35
|
+
* sent. This is the ordered proof: a hash of the whole array in its order. {@link SetProof} is the
|
|
36
|
+
* order-independent one a dataset uses.
|
|
37
|
+
*/
|
|
38
|
+
export type DatasetProof = {
|
|
39
|
+
/** `ordered`; absent on every proof made before the set proof existed */
|
|
40
|
+
kind?: 'ordered';
|
|
41
|
+
/** number of items in the collection */
|
|
42
|
+
count: number;
|
|
43
|
+
/** hash of the collection's canonical JSON */
|
|
44
|
+
digest: string;
|
|
45
|
+
};
|
|
46
|
+
/**
|
|
47
|
+
* The proof of a dataset: the number of rows and the sum of every row's hash modulo 2^256, as 64
|
|
48
|
+
* lowercase hex digits. A sum does not depend on order, so a patch moves it by the hashes of the rows
|
|
49
|
+
* it replaces and adds, and neither side reads the whole collection to check a sync. It detects
|
|
50
|
+
* corruption, a missed apply and a half-applied patch. It is not a defence against a malicious
|
|
51
|
+
* server, which can send rows and a digest that agree.
|
|
52
|
+
*/
|
|
53
|
+
export type SetProof = {
|
|
54
|
+
kind: 'set';
|
|
55
|
+
/** number of rows in the collection */
|
|
56
|
+
count: number;
|
|
57
|
+
/** sum of the row hashes modulo 2^256, 64 hex digits */
|
|
58
|
+
digest: string;
|
|
59
|
+
};
|
|
60
|
+
/** the page of items a query selects, with the size of the filtered collection and the page count */
|
|
61
|
+
export type QueryResult = {
|
|
62
|
+
/** the items on this page */
|
|
63
|
+
items: Record<string, unknown>[];
|
|
64
|
+
/** items matching the filters, before paging */
|
|
65
|
+
total: number;
|
|
66
|
+
/** the 1-based page returned */
|
|
67
|
+
page: number;
|
|
68
|
+
/** number of pages, at least 1 */
|
|
69
|
+
pages: number;
|
|
70
|
+
};
|
|
71
|
+
/** what a template or renderer receives as `query` for a query URL */
|
|
72
|
+
export type QueryView = {
|
|
73
|
+
/** items matching the filters, before paging */
|
|
74
|
+
total: number;
|
|
75
|
+
/** the 1-based page returned */
|
|
76
|
+
page: number;
|
|
77
|
+
/** number of pages, at least 1 */
|
|
78
|
+
pages: number;
|
|
79
|
+
};
|
|
80
|
+
/** the refusal reason when the stored items no longer match their proof */
|
|
81
|
+
export declare const PROOF_MISMATCH = "dataset proof does not match its items";
|
|
82
|
+
/** a flat object of a collection or the data around it */
|
|
83
|
+
export type Plain = Record<string, unknown>;
|
|
84
|
+
/** JSON with object keys sorted and no whitespace, so equal values give equal text on any engine */
|
|
85
|
+
export declare function canonicalJson(value: unknown): string;
|
|
86
|
+
/** the proof a query route's `load` result carries for its collection */
|
|
87
|
+
export declare function datasetProof(items: readonly unknown[]): Promise<DatasetProof>;
|
|
88
|
+
/** what a dataset's name may be; it never holds a `/`, which a row id uses to split name from key */
|
|
89
|
+
export declare const DATASET_NAME: RegExp;
|
|
90
|
+
/** the digest of an empty collection */
|
|
91
|
+
export declare const EMPTY_DIGEST: string;
|
|
92
|
+
/** the hash of one row: base64url SHA-256 of its key, a NUL, and its canonical JSON */
|
|
93
|
+
export declare function rowHash(key: string, row: unknown): Promise<string>;
|
|
94
|
+
/** `digest` plus the row hash `hash`, modulo 2^256 */
|
|
95
|
+
export declare function digestAdd(digest: string, hash: string): string;
|
|
96
|
+
/** `digest` minus the row hash `hash`, modulo 2^256 */
|
|
97
|
+
export declare function digestSub(digest: string, hash: string): string;
|
|
98
|
+
/** the set proof of rows by key; read every row once, so a patch should update it with {@link digestAdd} and {@link digestSub} */
|
|
99
|
+
export declare function setProof(rows: ReadonlyMap<string, unknown>): Promise<SetProof>;
|
|
100
|
+
/** whether `value` is shaped like a set proof */
|
|
101
|
+
export declare function isSetProof(value: unknown): value is SetProof;
|
|
102
|
+
/** reads the key of a row for a set proof; throws or returns a non-string for a row it cannot key */
|
|
103
|
+
export type KeyOf = (row: Plain) => string;
|
|
104
|
+
/** the {@link KeyOf} a route's `query.key` names: the item's own field, which must be a string */
|
|
105
|
+
export declare function keyField(schema: QuerySchema): KeyOf | undefined;
|
|
106
|
+
/**
|
|
107
|
+
* Whether `data` holds the collection at `schema.list` and a `proof` that matches it. A proof with
|
|
108
|
+
* `kind: 'set'` is checked with `keyOf`, which reads each row's key; without one it fails.
|
|
109
|
+
*/
|
|
110
|
+
export declare function verifyProof(schema: QuerySchema, data: unknown, keyOf?: KeyOf): Promise<boolean>;
|
|
111
|
+
/**
|
|
112
|
+
* Applies a query string to the collection in `data`: filters by equality, sorts stably (ties keep
|
|
113
|
+
* the stored order), and pages. Returns `null` for anything the schema does not declare, so the
|
|
114
|
+
* caller goes to the network: an unknown parameter, a number filter that is not a finite number, a
|
|
115
|
+
* `sort` that is repeated or not a listed field, a `page` that is not a positive integer or lies
|
|
116
|
+
* past the last, or an item that is not an object. An empty query string selects everything.
|
|
117
|
+
*/
|
|
118
|
+
export declare function evaluateQuery(schema: QuerySchema, data: unknown, search: string): QueryResult | null;
|
|
119
|
+
/**
|
|
120
|
+
* The data a template or renderer gets for a query URL: the stored data with `schema.list` replaced
|
|
121
|
+
* by the page's items and a `query` key added. Refuses when the proof fails or the query is not one
|
|
122
|
+
* the schema declares.
|
|
123
|
+
*/
|
|
124
|
+
export declare function shapeQuery(schema: QuerySchema, data: unknown, search: string, keyOf?: KeyOf): Promise<Shaped>;
|
|
125
|
+
/** what {@link shapeQuery} and {@link shapeRows} return */
|
|
126
|
+
export type Shaped = {
|
|
127
|
+
/** true when the query was answered from the stored collection */
|
|
128
|
+
ok: true;
|
|
129
|
+
/** the stored data with the page's items and the query view */
|
|
130
|
+
data: Plain;
|
|
131
|
+
} | {
|
|
132
|
+
/** false when the proof or the query was refused */
|
|
133
|
+
ok: false;
|
|
134
|
+
/** why the query was refused */
|
|
135
|
+
reason: string;
|
|
136
|
+
};
|
|
137
|
+
/**
|
|
138
|
+
* {@link shapeQuery} without the proof check, for a collection the caller has already verified,
|
|
139
|
+
* such as the rows of a held dataset.
|
|
140
|
+
*/
|
|
141
|
+
export declare function shapeRows(schema: QuerySchema, data: unknown, search: string): Shaped;
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import { shellHash } from './hash.js';
|
|
2
|
+
/** the refusal reason when the stored items no longer match their proof */
|
|
3
|
+
export const PROOF_MISMATCH = 'dataset proof does not match its items';
|
|
4
|
+
const isPlain = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
5
|
+
const own = (value, key) => Object.hasOwn(value, key) ? value[key] : undefined;
|
|
6
|
+
function canonical(value) {
|
|
7
|
+
const convert = value?.toJSON;
|
|
8
|
+
if (typeof value === 'object' && typeof convert === 'function') {
|
|
9
|
+
return canonical(convert.call(value));
|
|
10
|
+
}
|
|
11
|
+
if (Array.isArray(value))
|
|
12
|
+
return `[${value.map((v) => canonical(v) ?? 'null').join(',')}]`;
|
|
13
|
+
if (isPlain(value)) {
|
|
14
|
+
const parts = [];
|
|
15
|
+
for (const key of Object.keys(value).sort()) {
|
|
16
|
+
const part = canonical(value[key]);
|
|
17
|
+
if (part !== undefined)
|
|
18
|
+
parts.push(`${JSON.stringify(key)}:${part}`);
|
|
19
|
+
}
|
|
20
|
+
return `{${parts.join(',')}}`;
|
|
21
|
+
}
|
|
22
|
+
return JSON.stringify(value);
|
|
23
|
+
}
|
|
24
|
+
/** JSON with object keys sorted and no whitespace, so equal values give equal text on any engine */
|
|
25
|
+
export function canonicalJson(value) {
|
|
26
|
+
return canonical(value) ?? 'null';
|
|
27
|
+
}
|
|
28
|
+
/** the proof a query route's `load` result carries for its collection */
|
|
29
|
+
export async function datasetProof(items) {
|
|
30
|
+
return { count: items.length, digest: await shellHash(canonicalJson(items)) };
|
|
31
|
+
}
|
|
32
|
+
/** what a dataset's name may be; it never holds a `/`, which a row id uses to split name from key */
|
|
33
|
+
export const DATASET_NAME = /^[A-Za-z0-9._:-]{1,128}$/;
|
|
34
|
+
const MODULUS = 1n << 256n;
|
|
35
|
+
/** the digest of an empty collection */
|
|
36
|
+
export const EMPTY_DIGEST = '0'.repeat(64);
|
|
37
|
+
const DIGEST_FORM = /^[0-9a-f]{64}$/;
|
|
38
|
+
/** the hash of one row: base64url SHA-256 of its key, a NUL, and its canonical JSON */
|
|
39
|
+
export function rowHash(key, row) {
|
|
40
|
+
return shellHash(`${key}\0${canonicalJson(row)}`);
|
|
41
|
+
}
|
|
42
|
+
function hashValue(hash) {
|
|
43
|
+
if (!/^[A-Za-z0-9_-]{43}$/.test(hash))
|
|
44
|
+
throw new RangeError('not a row hash');
|
|
45
|
+
const binary = atob(hash.replace(/-/g, '+').replace(/_/g, '/') + '=');
|
|
46
|
+
let hex = '';
|
|
47
|
+
for (let i = 0; i < binary.length; i++)
|
|
48
|
+
hex += binary.charCodeAt(i).toString(16).padStart(2, '0');
|
|
49
|
+
return BigInt(`0x${hex}`);
|
|
50
|
+
}
|
|
51
|
+
function digestValue(digest) {
|
|
52
|
+
if (!DIGEST_FORM.test(digest))
|
|
53
|
+
throw new RangeError('not a set digest');
|
|
54
|
+
return BigInt(`0x${digest}`);
|
|
55
|
+
}
|
|
56
|
+
const digestText = (value) => (((value % MODULUS) + MODULUS) % MODULUS).toString(16).padStart(64, '0');
|
|
57
|
+
/** `digest` plus the row hash `hash`, modulo 2^256 */
|
|
58
|
+
export function digestAdd(digest, hash) {
|
|
59
|
+
return digestText(digestValue(digest) + hashValue(hash));
|
|
60
|
+
}
|
|
61
|
+
/** `digest` minus the row hash `hash`, modulo 2^256 */
|
|
62
|
+
export function digestSub(digest, hash) {
|
|
63
|
+
return digestText(digestValue(digest) - hashValue(hash));
|
|
64
|
+
}
|
|
65
|
+
/** the set proof of rows by key; read every row once, so a patch should update it with {@link digestAdd} and {@link digestSub} */
|
|
66
|
+
export async function setProof(rows) {
|
|
67
|
+
const hashes = await Promise.all([...rows].map(([key, row]) => rowHash(key, row)));
|
|
68
|
+
let sum = 0n;
|
|
69
|
+
for (const hash of hashes)
|
|
70
|
+
sum += hashValue(hash);
|
|
71
|
+
return { kind: 'set', count: rows.size, digest: digestText(sum) };
|
|
72
|
+
}
|
|
73
|
+
/** whether `value` is shaped like a set proof */
|
|
74
|
+
export function isSetProof(value) {
|
|
75
|
+
return (isPlain(value) &&
|
|
76
|
+
value['kind'] === 'set' &&
|
|
77
|
+
typeof value['count'] === 'number' &&
|
|
78
|
+
Number.isSafeInteger(value['count']) &&
|
|
79
|
+
value['count'] >= 0 &&
|
|
80
|
+
typeof value['digest'] === 'string' &&
|
|
81
|
+
DIGEST_FORM.test(value['digest']));
|
|
82
|
+
}
|
|
83
|
+
/** the {@link KeyOf} a route's `query.key` names: the item's own field, which must be a string */
|
|
84
|
+
export function keyField(schema) {
|
|
85
|
+
const field = schema.key;
|
|
86
|
+
if (field === undefined)
|
|
87
|
+
return undefined;
|
|
88
|
+
return (row) => {
|
|
89
|
+
const key = own(row, field);
|
|
90
|
+
if (typeof key !== 'string')
|
|
91
|
+
throw new TypeError(`${field} is not a string key`);
|
|
92
|
+
return key;
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
async function verifySet(items, proof, keyOf) {
|
|
96
|
+
if (proof.count !== items.length)
|
|
97
|
+
return false;
|
|
98
|
+
const rows = new Map();
|
|
99
|
+
for (const item of items) {
|
|
100
|
+
if (!isPlain(item))
|
|
101
|
+
return false;
|
|
102
|
+
let key;
|
|
103
|
+
try {
|
|
104
|
+
key = keyOf(item);
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
109
|
+
if (typeof key !== 'string' || rows.has(key))
|
|
110
|
+
return false;
|
|
111
|
+
rows.set(key, item);
|
|
112
|
+
}
|
|
113
|
+
return (await setProof(rows)).digest === proof.digest;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Whether `data` holds the collection at `schema.list` and a `proof` that matches it. A proof with
|
|
117
|
+
* `kind: 'set'` is checked with `keyOf`, which reads each row's key; without one it fails.
|
|
118
|
+
*/
|
|
119
|
+
export async function verifyProof(schema, data, keyOf) {
|
|
120
|
+
if (!isPlain(data))
|
|
121
|
+
return false;
|
|
122
|
+
const items = own(data, schema.list);
|
|
123
|
+
const proof = own(data, 'proof');
|
|
124
|
+
if (!Array.isArray(items) || !isPlain(proof))
|
|
125
|
+
return false;
|
|
126
|
+
const kind = proof['kind'];
|
|
127
|
+
if (kind === 'set') {
|
|
128
|
+
return keyOf !== undefined && isSetProof(proof) && (await verifySet(items, proof, keyOf));
|
|
129
|
+
}
|
|
130
|
+
if (kind !== undefined && kind !== 'ordered')
|
|
131
|
+
return false;
|
|
132
|
+
if (proof['count'] !== items.length || typeof proof['digest'] !== 'string')
|
|
133
|
+
return false;
|
|
134
|
+
return (await shellHash(canonicalJson(items))) === proof['digest'];
|
|
135
|
+
}
|
|
136
|
+
const numberOf = (text) => text.trim() !== '' && Number.isFinite(Number(text)) ? Number(text) : null;
|
|
137
|
+
const rank = (value) => typeof value === 'number' ? 0 : typeof value === 'string' ? 1 : 2;
|
|
138
|
+
/** numbers before strings before anything else; numbers by value, strings by code unit */
|
|
139
|
+
function compare(a, b) {
|
|
140
|
+
const [ra, rb] = [rank(a), rank(b)];
|
|
141
|
+
if (ra !== rb)
|
|
142
|
+
return ra - rb;
|
|
143
|
+
if (ra === 2)
|
|
144
|
+
return 0;
|
|
145
|
+
return a < b
|
|
146
|
+
? -1
|
|
147
|
+
: a > b
|
|
148
|
+
? 1
|
|
149
|
+
: 0;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Applies a query string to the collection in `data`: filters by equality, sorts stably (ties keep
|
|
153
|
+
* the stored order), and pages. Returns `null` for anything the schema does not declare, so the
|
|
154
|
+
* caller goes to the network: an unknown parameter, a number filter that is not a finite number, a
|
|
155
|
+
* `sort` that is repeated or not a listed field, a `page` that is not a positive integer or lies
|
|
156
|
+
* past the last, or an item that is not an object. An empty query string selects everything.
|
|
157
|
+
*/
|
|
158
|
+
export function evaluateQuery(schema, data, search) {
|
|
159
|
+
const list = isPlain(data) ? own(data, schema.list) : undefined;
|
|
160
|
+
if (!Array.isArray(list) || !list.every(isPlain))
|
|
161
|
+
return null;
|
|
162
|
+
const params = new URLSearchParams(search);
|
|
163
|
+
const filter = schema.filter ?? {};
|
|
164
|
+
for (const key of new Set(params.keys())) {
|
|
165
|
+
if (key !== 'sort' && key !== 'page' && !Object.hasOwn(filter, key))
|
|
166
|
+
return null;
|
|
167
|
+
}
|
|
168
|
+
let items = list;
|
|
169
|
+
for (const [field, type] of Object.entries(filter)) {
|
|
170
|
+
const wanted = params.getAll(field);
|
|
171
|
+
if (wanted.length === 0)
|
|
172
|
+
continue;
|
|
173
|
+
if (type === 'number') {
|
|
174
|
+
const numbers = wanted.map(numberOf);
|
|
175
|
+
if (numbers.includes(null))
|
|
176
|
+
return null;
|
|
177
|
+
items = items.filter((item) => {
|
|
178
|
+
const value = own(item, field);
|
|
179
|
+
return typeof value === 'number' && numbers.includes(value);
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
items = items.filter((item) => {
|
|
184
|
+
const value = own(item, field);
|
|
185
|
+
const text = typeof value === 'string' ||
|
|
186
|
+
typeof value === 'number' ||
|
|
187
|
+
typeof value === 'boolean'
|
|
188
|
+
? String(value)
|
|
189
|
+
: undefined;
|
|
190
|
+
return text !== undefined && wanted.includes(text);
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
const sorts = params.getAll('sort');
|
|
195
|
+
if (sorts.length > 1)
|
|
196
|
+
return null;
|
|
197
|
+
items = [...items];
|
|
198
|
+
if (sorts.length === 1) {
|
|
199
|
+
const raw = sorts[0];
|
|
200
|
+
const listed = schema.sort ?? [];
|
|
201
|
+
const descending = !listed.includes(raw) && raw.startsWith('-');
|
|
202
|
+
const field = descending ? raw.slice(1) : raw;
|
|
203
|
+
if (!listed.includes(field))
|
|
204
|
+
return null;
|
|
205
|
+
items.sort((a, b) => {
|
|
206
|
+
const order = compare(own(a, field), own(b, field));
|
|
207
|
+
return descending ? -order : order;
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
const pageParams = params.getAll('page');
|
|
211
|
+
if (pageParams.length > 1)
|
|
212
|
+
return null;
|
|
213
|
+
let page = 1;
|
|
214
|
+
if (pageParams.length === 1) {
|
|
215
|
+
if (!/^[1-9]\d{0,14}$/.test(pageParams[0]))
|
|
216
|
+
return null;
|
|
217
|
+
page = Number(pageParams[0]);
|
|
218
|
+
}
|
|
219
|
+
const total = items.length;
|
|
220
|
+
const size = schema.pageSize;
|
|
221
|
+
const pages = size === undefined ? 1 : Math.max(1, Math.ceil(total / size));
|
|
222
|
+
if (page > pages)
|
|
223
|
+
return null;
|
|
224
|
+
return {
|
|
225
|
+
items: size === undefined ? items : items.slice((page - 1) * size, page * size),
|
|
226
|
+
total,
|
|
227
|
+
page,
|
|
228
|
+
pages
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* The data a template or renderer gets for a query URL: the stored data with `schema.list` replaced
|
|
233
|
+
* by the page's items and a `query` key added. Refuses when the proof fails or the query is not one
|
|
234
|
+
* the schema declares.
|
|
235
|
+
*/
|
|
236
|
+
export async function shapeQuery(schema, data, search, keyOf) {
|
|
237
|
+
if (!(await verifyProof(schema, data, keyOf)))
|
|
238
|
+
return { ok: false, reason: PROOF_MISMATCH };
|
|
239
|
+
return shapeRows(schema, data, search);
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* {@link shapeQuery} without the proof check, for a collection the caller has already verified,
|
|
243
|
+
* such as the rows of a held dataset.
|
|
244
|
+
*/
|
|
245
|
+
export function shapeRows(schema, data, search) {
|
|
246
|
+
const result = evaluateQuery(schema, data, search);
|
|
247
|
+
if (!result)
|
|
248
|
+
return { ok: false, reason: 'query is not one the route declares' };
|
|
249
|
+
const { items, total, page, pages } = result;
|
|
250
|
+
const view = { total, page, pages };
|
|
251
|
+
return { ok: true, data: { ...data, [schema.list]: items, query: view } };
|
|
252
|
+
}
|