@drupflare/spool 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +263 -41
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/entry.js +33 -1
- package/dist/client/register.d.ts +23 -1
- package/dist/client/register.js +53 -7
- package/dist/protocol/config.d.ts +16 -1
- package/dist/protocol/config.js +3 -1
- package/dist/protocol/dataset.d.ts +183 -0
- package/dist/protocol/dataset.js +206 -0
- package/dist/protocol/envelope.d.ts +1 -0
- package/dist/protocol/envelope.js +2 -1
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/index.d.ts +3 -0
- package/dist/protocol/index.js +3 -0
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +46 -0
- package/dist/protocol/provision.js +38 -0
- package/dist/protocol/query.d.ts +66 -5
- package/dist/protocol/query.js +103 -4
- package/dist/protocol/route.d.ts +6 -2
- package/dist/protocol/route.js +31 -3
- package/dist/protocol/write.d.ts +146 -0
- package/dist/protocol/write.js +101 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +11 -1
- package/dist/report/economics.js +19 -1
- package/dist/report/index.d.ts +2 -0
- package/dist/report/index.js +2 -0
- package/dist/report/inputs.d.ts +97 -0
- package/dist/report/inputs.js +133 -0
- package/dist/report/institution.d.ts +188 -1
- package/dist/report/institution.js +146 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +3 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/dataset.d.ts +48 -0
- package/dist/server/dataset.js +308 -0
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +2 -0
- package/dist/server/spool.d.ts +20 -0
- package/dist/server/spool.js +126 -18
- package/dist/server/write.d.ts +27 -0
- package/dist/server/write.js +127 -0
- package/dist/sw/analytics.d.ts +6 -0
- package/dist/sw/analytics.js +14 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.js +78 -9
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/render.d.ts +7 -2
- package/dist/sw/render.js +19 -5
- package/dist/sw/replica.d.ts +12 -3
- package/dist/sw/replica.js +40 -5
- package/dist/sw/runtime.d.ts +43 -2
- package/dist/sw/runtime.js +340 -24
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +107 -0
- package/dist/sw/write.js +442 -0
- package/package.json +3 -2
package/dist/sw/replica.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Artifact } from '../protocol/artifact.js';
|
|
2
2
|
import { type Envelope } from '../protocol/envelope.js';
|
|
3
3
|
import { type SyncResponse } from '../protocol/sync.js';
|
|
4
|
+
import { DatasetStore, type DatasetStoreDeps } from './dataset.js';
|
|
4
5
|
import { Graph, type Materialized } from './graph/index.js';
|
|
5
6
|
import { type HeaderSet } from './policy.js';
|
|
6
7
|
import type { ReplicaStorage, StoredArtifact } from './storage.js';
|
|
@@ -14,7 +15,9 @@ export declare class Replica {
|
|
|
14
15
|
#private;
|
|
15
16
|
/** records and projections; untouched until a page that references them is stored or shown */
|
|
16
17
|
readonly graph: Graph;
|
|
17
|
-
|
|
18
|
+
/** the datasets held, apart from the version vector: each has its own cursor */
|
|
19
|
+
readonly datasets: DatasetStore;
|
|
20
|
+
constructor(storage: ReplicaStorage, version: string, now?: () => number, hash?: DatasetStoreDeps['hash']);
|
|
18
21
|
/** reads the persisted meta; a different runtime version wipes what the old one stored */
|
|
19
22
|
load(): Promise<void>;
|
|
20
23
|
/** the vector cursor the replica is current to, or null before the first artifact */
|
|
@@ -48,6 +51,8 @@ export declare class Replica {
|
|
|
48
51
|
needFor(artifact: Artifact, always?: boolean): Record<string, number> | null;
|
|
49
52
|
/** the records and projections held, for an artifact request's `have`; loads the graph */
|
|
50
53
|
held(): Promise<Record<string, number>>;
|
|
54
|
+
/** the datasets whose rows `artifact` references, directly or through projections; loads the graph when it has references */
|
|
55
|
+
datasetsOf(artifact: Artifact): Promise<string[]>;
|
|
51
56
|
/** the artifact with its references resolved from the graph; loads the graph when it has any */
|
|
52
57
|
materialize(artifact: Artifact): Promise<Materialized>;
|
|
53
58
|
/**
|
|
@@ -66,8 +71,12 @@ export declare class Replica {
|
|
|
66
71
|
sweep(need: Record<string, number>): Promise<Record<string, number> | null>;
|
|
67
72
|
/** removes artifacts by id */
|
|
68
73
|
drop(ids: readonly string[]): Promise<void>;
|
|
69
|
-
/**
|
|
70
|
-
|
|
74
|
+
/**
|
|
75
|
+
* Empties the replica; `cursor` is where it restarts. Datasets go too unless `keepDatasets`: a
|
|
76
|
+
* vector reset says nothing about a dataset, whose own cursor and horizon decide when it reseeds.
|
|
77
|
+
* Either way an apply that began before the wipe is dropped.
|
|
78
|
+
*/
|
|
79
|
+
wipe(cursor?: number | null, keepDatasets?: boolean): Promise<void>;
|
|
71
80
|
/** applies a `/.spool/sync` answer and returns the ids of the artifacts it invalidated */
|
|
72
81
|
applySync(response: SyncResponse): Promise<string[]>;
|
|
73
82
|
/**
|
package/dist/sw/replica.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { CLEAR_ALL } from '../protocol/envelope.js';
|
|
2
2
|
import { ARTIFACT_TAG_PREFIX, artifactTag, MAX_HAVE, MAX_NEED } from '../protocol/sync.js';
|
|
3
|
+
import { DatasetStore } from './dataset.js';
|
|
3
4
|
import { isCurrent, validatedAt } from './freshness.js';
|
|
4
5
|
import { Graph, usesGraph } from './graph/index.js';
|
|
5
6
|
import { MAX_POLICY_PATHS } from './policy.js';
|
|
@@ -23,16 +24,36 @@ export class Replica {
|
|
|
23
24
|
#policies = new Map();
|
|
24
25
|
/** records and projections; untouched until a page that references them is stored or shown */
|
|
25
26
|
graph;
|
|
26
|
-
|
|
27
|
+
/** the datasets held, apart from the version vector: each has its own cursor */
|
|
28
|
+
datasets;
|
|
29
|
+
constructor(storage, version, now = Date.now, hash) {
|
|
27
30
|
this.#storage = storage;
|
|
28
31
|
this.#version = version;
|
|
29
32
|
this.#now = now;
|
|
33
|
+
this.datasets = new DatasetStore({
|
|
34
|
+
storage,
|
|
35
|
+
now,
|
|
36
|
+
epoch: () => this.#epoch,
|
|
37
|
+
...(hash ? { hash } : {}),
|
|
38
|
+
onChange: (name, puts, deletes) => this.#rowsChanged(name, puts, deletes)
|
|
39
|
+
});
|
|
30
40
|
this.graph = new Graph({
|
|
31
41
|
storage,
|
|
32
42
|
tagVersion: (tag) => this.#tags.get(tag) ?? 0,
|
|
33
|
-
artifacts: () => this.#all()
|
|
43
|
+
artifacts: () => this.#all(),
|
|
44
|
+
row: (dataset, key) => this.datasets.row(dataset, key),
|
|
45
|
+
held: (dataset) => this.datasets.loaded(dataset),
|
|
46
|
+
loadDatasets: async (names) => void (await Promise.all(names.map((n) => this.datasets.get(n))))
|
|
34
47
|
});
|
|
35
48
|
}
|
|
49
|
+
/** repairs what references changed dataset rows: a deleted row drops the pages that used it */
|
|
50
|
+
async #rowsChanged(name, puts, deletes) {
|
|
51
|
+
if (!this.graph.loaded)
|
|
52
|
+
return;
|
|
53
|
+
const effect = await this.graph.rowsChanged(name, puts, deletes);
|
|
54
|
+
if (effect.dropped.artifacts.length > 0)
|
|
55
|
+
await this.drop(effect.dropped.artifacts);
|
|
56
|
+
}
|
|
36
57
|
/** reads the persisted meta; a different runtime version wipes what the old one stored */
|
|
37
58
|
async load() {
|
|
38
59
|
const meta = await this.#storage.getMeta();
|
|
@@ -46,6 +67,7 @@ export class Replica {
|
|
|
46
67
|
}
|
|
47
68
|
await this.#storage.clearArtifacts();
|
|
48
69
|
await this.graph.clear();
|
|
70
|
+
await this.datasets.clear();
|
|
49
71
|
await this.#save();
|
|
50
72
|
}
|
|
51
73
|
/** the vector cursor the replica is current to, or null before the first artifact */
|
|
@@ -133,6 +155,13 @@ export class Replica {
|
|
|
133
155
|
await this.#wake();
|
|
134
156
|
return this.graph.held(MAX_HAVE);
|
|
135
157
|
}
|
|
158
|
+
/** the datasets whose rows `artifact` references, directly or through projections; loads the graph when it has references */
|
|
159
|
+
async datasetsOf(artifact) {
|
|
160
|
+
if (!usesGraph(artifact))
|
|
161
|
+
return [];
|
|
162
|
+
await this.#wake();
|
|
163
|
+
return this.graph.datasetsOf(artifact);
|
|
164
|
+
}
|
|
136
165
|
/** the artifact with its references resolved from the graph; loads the graph when it has any */
|
|
137
166
|
async materialize(artifact) {
|
|
138
167
|
if (usesGraph(artifact))
|
|
@@ -211,12 +240,18 @@ export class Replica {
|
|
|
211
240
|
if (this.graph.loaded)
|
|
212
241
|
this.graph.forgetArtifacts(ids);
|
|
213
242
|
}
|
|
214
|
-
/**
|
|
215
|
-
|
|
243
|
+
/**
|
|
244
|
+
* Empties the replica; `cursor` is where it restarts. Datasets go too unless `keepDatasets`: a
|
|
245
|
+
* vector reset says nothing about a dataset, whose own cursor and horizon decide when it reseeds.
|
|
246
|
+
* Either way an apply that began before the wipe is dropped.
|
|
247
|
+
*/
|
|
248
|
+
async wipe(cursor = null, keepDatasets = false) {
|
|
216
249
|
this.#epoch += 1;
|
|
217
250
|
this.#memory.clear();
|
|
218
251
|
await this.#storage.clearArtifacts();
|
|
219
252
|
await this.graph.clear();
|
|
253
|
+
if (!keepDatasets)
|
|
254
|
+
await this.datasets.clear();
|
|
220
255
|
this.#tags = new Map();
|
|
221
256
|
this.#site = undefined;
|
|
222
257
|
this.#policies = new Map();
|
|
@@ -227,7 +262,7 @@ export class Replica {
|
|
|
227
262
|
async applySync(response) {
|
|
228
263
|
if (response.reset) {
|
|
229
264
|
const held = await this.ids();
|
|
230
|
-
await this.wipe(response.cursor);
|
|
265
|
+
await this.wipe(response.cursor, true);
|
|
231
266
|
this.#syncedAt = this.#now();
|
|
232
267
|
await this.#save();
|
|
233
268
|
return held;
|
package/dist/sw/runtime.d.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { type AssetMode, type SwConfig } from '../protocol/config.js';
|
|
2
|
+
import { type WriteFlush, type WriteQueued, type WriteRefusal, type WriteResolved, type WriteStatus } from '../protocol/write.js';
|
|
3
|
+
import { type DatasetRefusal } from './dataset-sync.js';
|
|
2
4
|
import type { PartitionHost } from './partition.js';
|
|
3
5
|
import { Ledger, type LedgerEntry, type LedgerTotals } from './provision.js';
|
|
4
6
|
import { type Renderers } from './render.js';
|
|
5
7
|
import { Replica } from './replica.js';
|
|
6
8
|
import { type RoutableInstall } from './routing.js';
|
|
7
|
-
import type { ReplicaStorage } from './storage.js';
|
|
9
|
+
import type { DatasetMeta, ReplicaStorage } from './storage.js';
|
|
8
10
|
/** the slice of a `Cache` the runtime uses */
|
|
9
11
|
export type CacheLike = {
|
|
10
12
|
match(request: Request): Promise<Response | undefined>;
|
|
@@ -39,6 +41,11 @@ export type RuntimeDeps = {
|
|
|
39
41
|
partitions?: PartitionHost;
|
|
40
42
|
/** tells every page of the origin something, such as that a wipe happened */
|
|
41
43
|
broadcast?(message: unknown): void;
|
|
44
|
+
/** the storage the origin may use and has used; default `navigator.storage.estimate` */
|
|
45
|
+
estimate?(): Promise<{
|
|
46
|
+
quota?: number;
|
|
47
|
+
usage?: number;
|
|
48
|
+
}>;
|
|
42
49
|
};
|
|
43
50
|
/** the worker's answer to a page's `spool:hello` */
|
|
44
51
|
export type Hello = {
|
|
@@ -80,6 +87,8 @@ export type PartitionStatus = {
|
|
|
80
87
|
artifacts: string[];
|
|
81
88
|
records: string[];
|
|
82
89
|
projections: string[];
|
|
90
|
+
/** the principal datasets held for this partition; always empty while the flag is off */
|
|
91
|
+
datasets: DatasetMeta[];
|
|
83
92
|
};
|
|
84
93
|
/** the worker's answer to `spool:assets`: the mode asked for and whether Static Routing took effect */
|
|
85
94
|
export type AssetsStatus = {
|
|
@@ -93,8 +102,40 @@ export type HeldStatus = {
|
|
|
93
102
|
dropped: Record<string, number>;
|
|
94
103
|
third: string[];
|
|
95
104
|
};
|
|
105
|
+
/**
|
|
106
|
+
* the worker's answer to `spool:datasets`: what it holds, how each sync ended (counts by
|
|
107
|
+
* `<name>:<status>` or `<name>:refused:<reason>`), the datasets it will not retry this run, the
|
|
108
|
+
* datasets it found damaged and cleared, and whether a page was granted persistent storage
|
|
109
|
+
*/
|
|
110
|
+
export type DatasetStatus = {
|
|
111
|
+
type: 'spool:datasets';
|
|
112
|
+
datasets: DatasetMeta[];
|
|
113
|
+
outcomes: Record<string, number>;
|
|
114
|
+
refused: Record<string, DatasetRefusal>;
|
|
115
|
+
evicted: number;
|
|
116
|
+
persisted: boolean | null;
|
|
117
|
+
};
|
|
118
|
+
/** the worker's answer to `spool:write`, `spool:flush`, `spool:writes` and `spool:resolve` */
|
|
119
|
+
export type WriteReply = {
|
|
120
|
+
type: 'spool:write';
|
|
121
|
+
result: WriteQueued | WriteRefusal;
|
|
122
|
+
} | {
|
|
123
|
+
type: 'spool:flush';
|
|
124
|
+
result: ({
|
|
125
|
+
ok: true;
|
|
126
|
+
} & WriteFlush) | WriteRefusal;
|
|
127
|
+
} | {
|
|
128
|
+
type: 'spool:writes';
|
|
129
|
+
result: {
|
|
130
|
+
ok: true;
|
|
131
|
+
writes: WriteStatus[];
|
|
132
|
+
} | WriteRefusal;
|
|
133
|
+
} | {
|
|
134
|
+
type: 'spool:resolve';
|
|
135
|
+
result: WriteResolved | WriteRefusal;
|
|
136
|
+
};
|
|
96
137
|
/** every reply the worker gives a page */
|
|
97
|
-
export type Reply = Hello | Status | ViewId | GraphStatus | LedgerStatus | PartitionStatus | AssetsStatus | HeldStatus;
|
|
138
|
+
export type Reply = WriteReply | Hello | Status | ViewId | DatasetStatus | GraphStatus | LedgerStatus | PartitionStatus | AssetsStatus | HeldStatus;
|
|
98
139
|
/** the object `serve` wires to service worker events */
|
|
99
140
|
export type Runtime = {
|
|
100
141
|
/** a response promise for a request the worker handles, `undefined` for one it leaves alone */
|