@drupflare/spool 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +263 -41
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/entry.js +33 -1
  6. package/dist/client/register.d.ts +23 -1
  7. package/dist/client/register.js +53 -7
  8. package/dist/protocol/config.d.ts +16 -1
  9. package/dist/protocol/config.js +3 -1
  10. package/dist/protocol/dataset.d.ts +183 -0
  11. package/dist/protocol/dataset.js +206 -0
  12. package/dist/protocol/envelope.d.ts +1 -0
  13. package/dist/protocol/envelope.js +2 -1
  14. package/dist/protocol/errors.d.ts +1 -1
  15. package/dist/protocol/graph.d.ts +23 -4
  16. package/dist/protocol/graph.js +34 -3
  17. package/dist/protocol/index.d.ts +3 -0
  18. package/dist/protocol/index.js +3 -0
  19. package/dist/protocol/plan.d.ts +100 -0
  20. package/dist/protocol/plan.js +661 -0
  21. package/dist/protocol/provision.d.ts +46 -0
  22. package/dist/protocol/provision.js +38 -0
  23. package/dist/protocol/query.d.ts +66 -5
  24. package/dist/protocol/query.js +103 -4
  25. package/dist/protocol/route.d.ts +6 -2
  26. package/dist/protocol/route.js +31 -3
  27. package/dist/protocol/write.d.ts +146 -0
  28. package/dist/protocol/write.js +101 -0
  29. package/dist/report/dataset.d.ts +48 -0
  30. package/dist/report/dataset.js +93 -0
  31. package/dist/report/derive.d.ts +285 -0
  32. package/dist/report/derive.js +676 -1
  33. package/dist/report/economics.d.ts +11 -1
  34. package/dist/report/economics.js +19 -1
  35. package/dist/report/index.d.ts +2 -0
  36. package/dist/report/index.js +2 -0
  37. package/dist/report/inputs.d.ts +97 -0
  38. package/dist/report/inputs.js +133 -0
  39. package/dist/report/institution.d.ts +188 -1
  40. package/dist/report/institution.js +146 -0
  41. package/dist/report/loops.d.ts +73 -0
  42. package/dist/report/loops.js +101 -0
  43. package/dist/report/measurement.d.ts +1 -1
  44. package/dist/report/measurement.js +3 -1
  45. package/dist/report/model.d.ts +2 -0
  46. package/dist/report/model.js +4 -0
  47. package/dist/report/thirdparty.d.ts +57 -0
  48. package/dist/report/thirdparty.js +111 -0
  49. package/dist/report/trim.d.ts +56 -0
  50. package/dist/report/trim.js +83 -1
  51. package/dist/server/dataset.d.ts +48 -0
  52. package/dist/server/dataset.js +308 -0
  53. package/dist/server/embedded.d.ts +3 -3
  54. package/dist/server/embedded.js +3 -3
  55. package/dist/server/index.d.ts +2 -0
  56. package/dist/server/index.js +2 -0
  57. package/dist/server/spool.d.ts +20 -0
  58. package/dist/server/spool.js +126 -18
  59. package/dist/server/write.d.ts +27 -0
  60. package/dist/server/write.js +127 -0
  61. package/dist/sw/analytics.d.ts +6 -0
  62. package/dist/sw/analytics.js +14 -0
  63. package/dist/sw/dataset-sync.d.ts +76 -0
  64. package/dist/sw/dataset-sync.js +149 -0
  65. package/dist/sw/dataset.d.ts +91 -0
  66. package/dist/sw/dataset.js +318 -0
  67. package/dist/sw/graph/index.d.ts +31 -1
  68. package/dist/sw/graph/index.js +57 -5
  69. package/dist/sw/graph/invalidate.d.ts +15 -3
  70. package/dist/sw/graph/invalidate.js +42 -4
  71. package/dist/sw/idb.js +78 -9
  72. package/dist/sw/partition.d.ts +6 -1
  73. package/dist/sw/partition.js +21 -2
  74. package/dist/sw/principal-dataset.d.ts +64 -0
  75. package/dist/sw/principal-dataset.js +254 -0
  76. package/dist/sw/render.d.ts +7 -2
  77. package/dist/sw/render.js +19 -5
  78. package/dist/sw/replica.d.ts +12 -3
  79. package/dist/sw/replica.js +40 -5
  80. package/dist/sw/runtime.d.ts +43 -2
  81. package/dist/sw/runtime.js +340 -24
  82. package/dist/sw/seal.d.ts +13 -0
  83. package/dist/sw/seal.js +13 -4
  84. package/dist/sw/static.d.ts +18 -0
  85. package/dist/sw/static.js +84 -1
  86. package/dist/sw/storage.d.ts +51 -1
  87. package/dist/sw/storage.js +33 -1
  88. package/dist/sw/write.d.ts +107 -0
  89. package/dist/sw/write.js +442 -0
  90. package/package.json +3 -2
@@ -1,6 +1,7 @@
1
1
  import type { Artifact } from '../protocol/artifact.js';
2
2
  import { type Envelope } from '../protocol/envelope.js';
3
3
  import { type SyncResponse } from '../protocol/sync.js';
4
+ import { DatasetStore, type DatasetStoreDeps } from './dataset.js';
4
5
  import { Graph, type Materialized } from './graph/index.js';
5
6
  import { type HeaderSet } from './policy.js';
6
7
  import type { ReplicaStorage, StoredArtifact } from './storage.js';
@@ -14,7 +15,9 @@ export declare class Replica {
14
15
  #private;
15
16
  /** records and projections; untouched until a page that references them is stored or shown */
16
17
  readonly graph: Graph;
17
- constructor(storage: ReplicaStorage, version: string, now?: () => number);
18
+ /** the datasets held, apart from the version vector: each has its own cursor */
19
+ readonly datasets: DatasetStore;
20
+ constructor(storage: ReplicaStorage, version: string, now?: () => number, hash?: DatasetStoreDeps['hash']);
18
21
  /** reads the persisted meta; a different runtime version wipes what the old one stored */
19
22
  load(): Promise<void>;
20
23
  /** the vector cursor the replica is current to, or null before the first artifact */
@@ -48,6 +51,8 @@ export declare class Replica {
48
51
  needFor(artifact: Artifact, always?: boolean): Record<string, number> | null;
49
52
  /** the records and projections held, for an artifact request's `have`; loads the graph */
50
53
  held(): Promise<Record<string, number>>;
54
+ /** the datasets whose rows `artifact` references, directly or through projections; loads the graph when it has references */
55
+ datasetsOf(artifact: Artifact): Promise<string[]>;
51
56
  /** the artifact with its references resolved from the graph; loads the graph when it has any */
52
57
  materialize(artifact: Artifact): Promise<Materialized>;
53
58
  /**
@@ -66,8 +71,12 @@ export declare class Replica {
66
71
  sweep(need: Record<string, number>): Promise<Record<string, number> | null>;
67
72
  /** removes artifacts by id */
68
73
  drop(ids: readonly string[]): Promise<void>;
69
- /** empties the replica; `cursor` is where it restarts */
70
- wipe(cursor?: number | null): Promise<void>;
74
+ /**
75
+ * Empties the replica; `cursor` is where it restarts. Datasets go too unless `keepDatasets`: a
76
+ * vector reset says nothing about a dataset, whose own cursor and horizon decide when it reseeds.
77
+ * Either way an apply that began before the wipe is dropped.
78
+ */
79
+ wipe(cursor?: number | null, keepDatasets?: boolean): Promise<void>;
71
80
  /** applies a `/.spool/sync` answer and returns the ids of the artifacts it invalidated */
72
81
  applySync(response: SyncResponse): Promise<string[]>;
73
82
  /**
@@ -1,5 +1,6 @@
1
1
  import { CLEAR_ALL } from '../protocol/envelope.js';
2
2
  import { ARTIFACT_TAG_PREFIX, artifactTag, MAX_HAVE, MAX_NEED } from '../protocol/sync.js';
3
+ import { DatasetStore } from './dataset.js';
3
4
  import { isCurrent, validatedAt } from './freshness.js';
4
5
  import { Graph, usesGraph } from './graph/index.js';
5
6
  import { MAX_POLICY_PATHS } from './policy.js';
@@ -23,16 +24,36 @@ export class Replica {
23
24
  #policies = new Map();
24
25
  /** records and projections; untouched until a page that references them is stored or shown */
25
26
  graph;
26
- constructor(storage, version, now = Date.now) {
27
+ /** the datasets held, apart from the version vector: each has its own cursor */
28
+ datasets;
29
+ constructor(storage, version, now = Date.now, hash) {
27
30
  this.#storage = storage;
28
31
  this.#version = version;
29
32
  this.#now = now;
33
+ this.datasets = new DatasetStore({
34
+ storage,
35
+ now,
36
+ epoch: () => this.#epoch,
37
+ ...(hash ? { hash } : {}),
38
+ onChange: (name, puts, deletes) => this.#rowsChanged(name, puts, deletes)
39
+ });
30
40
  this.graph = new Graph({
31
41
  storage,
32
42
  tagVersion: (tag) => this.#tags.get(tag) ?? 0,
33
- artifacts: () => this.#all()
43
+ artifacts: () => this.#all(),
44
+ row: (dataset, key) => this.datasets.row(dataset, key),
45
+ held: (dataset) => this.datasets.loaded(dataset),
46
+ loadDatasets: async (names) => void (await Promise.all(names.map((n) => this.datasets.get(n))))
34
47
  });
35
48
  }
49
+ /** repairs what references changed dataset rows: a deleted row drops the pages that used it */
50
+ async #rowsChanged(name, puts, deletes) {
51
+ if (!this.graph.loaded)
52
+ return;
53
+ const effect = await this.graph.rowsChanged(name, puts, deletes);
54
+ if (effect.dropped.artifacts.length > 0)
55
+ await this.drop(effect.dropped.artifacts);
56
+ }
36
57
  /** reads the persisted meta; a different runtime version wipes what the old one stored */
37
58
  async load() {
38
59
  const meta = await this.#storage.getMeta();
@@ -46,6 +67,7 @@ export class Replica {
46
67
  }
47
68
  await this.#storage.clearArtifacts();
48
69
  await this.graph.clear();
70
+ await this.datasets.clear();
49
71
  await this.#save();
50
72
  }
51
73
  /** the vector cursor the replica is current to, or null before the first artifact */
@@ -133,6 +155,13 @@ export class Replica {
133
155
  await this.#wake();
134
156
  return this.graph.held(MAX_HAVE);
135
157
  }
158
+ /** the datasets whose rows `artifact` references, directly or through projections; loads the graph when it has references */
159
+ async datasetsOf(artifact) {
160
+ if (!usesGraph(artifact))
161
+ return [];
162
+ await this.#wake();
163
+ return this.graph.datasetsOf(artifact);
164
+ }
136
165
  /** the artifact with its references resolved from the graph; loads the graph when it has any */
137
166
  async materialize(artifact) {
138
167
  if (usesGraph(artifact))
@@ -211,12 +240,18 @@ export class Replica {
211
240
  if (this.graph.loaded)
212
241
  this.graph.forgetArtifacts(ids);
213
242
  }
214
- /** empties the replica; `cursor` is where it restarts */
215
- async wipe(cursor = null) {
243
+ /**
244
+ * Empties the replica; `cursor` is where it restarts. Datasets go too unless `keepDatasets`: a
245
+ * vector reset says nothing about a dataset, whose own cursor and horizon decide when it reseeds.
246
+ * Either way an apply that began before the wipe is dropped.
247
+ */
248
+ async wipe(cursor = null, keepDatasets = false) {
216
249
  this.#epoch += 1;
217
250
  this.#memory.clear();
218
251
  await this.#storage.clearArtifacts();
219
252
  await this.graph.clear();
253
+ if (!keepDatasets)
254
+ await this.datasets.clear();
220
255
  this.#tags = new Map();
221
256
  this.#site = undefined;
222
257
  this.#policies = new Map();
@@ -227,7 +262,7 @@ export class Replica {
227
262
  async applySync(response) {
228
263
  if (response.reset) {
229
264
  const held = await this.ids();
230
- await this.wipe(response.cursor);
265
+ await this.wipe(response.cursor, true);
231
266
  this.#syncedAt = this.#now();
232
267
  await this.#save();
233
268
  return held;
@@ -1,10 +1,12 @@
1
1
  import { type AssetMode, type SwConfig } from '../protocol/config.js';
2
+ import { type WriteFlush, type WriteQueued, type WriteRefusal, type WriteResolved, type WriteStatus } from '../protocol/write.js';
3
+ import { type DatasetRefusal } from './dataset-sync.js';
2
4
  import type { PartitionHost } from './partition.js';
3
5
  import { Ledger, type LedgerEntry, type LedgerTotals } from './provision.js';
4
6
  import { type Renderers } from './render.js';
5
7
  import { Replica } from './replica.js';
6
8
  import { type RoutableInstall } from './routing.js';
7
- import type { ReplicaStorage } from './storage.js';
9
+ import type { DatasetMeta, ReplicaStorage } from './storage.js';
8
10
  /** the slice of a `Cache` the runtime uses */
9
11
  export type CacheLike = {
10
12
  match(request: Request): Promise<Response | undefined>;
@@ -39,6 +41,11 @@ export type RuntimeDeps = {
39
41
  partitions?: PartitionHost;
40
42
  /** tells every page of the origin something, such as that a wipe happened */
41
43
  broadcast?(message: unknown): void;
44
+ /** the storage the origin may use and has used; default `navigator.storage.estimate` */
45
+ estimate?(): Promise<{
46
+ quota?: number;
47
+ usage?: number;
48
+ }>;
42
49
  };
43
50
  /** the worker's answer to a page's `spool:hello` */
44
51
  export type Hello = {
@@ -80,6 +87,8 @@ export type PartitionStatus = {
80
87
  artifacts: string[];
81
88
  records: string[];
82
89
  projections: string[];
90
+ /** the principal datasets held for this partition; always empty while the flag is off */
91
+ datasets: DatasetMeta[];
83
92
  };
84
93
  /** the worker's answer to `spool:assets`: the mode asked for and whether Static Routing took effect */
85
94
  export type AssetsStatus = {
@@ -93,8 +102,40 @@ export type HeldStatus = {
93
102
  dropped: Record<string, number>;
94
103
  third: string[];
95
104
  };
105
+ /**
106
+ * the worker's answer to `spool:datasets`: what it holds, how each sync ended (counts by
107
+ * `<name>:<status>` or `<name>:refused:<reason>`), the datasets it will not retry this run, the
108
+ * datasets it found damaged and cleared, and whether a page was granted persistent storage
109
+ */
110
+ export type DatasetStatus = {
111
+ type: 'spool:datasets';
112
+ datasets: DatasetMeta[];
113
+ outcomes: Record<string, number>;
114
+ refused: Record<string, DatasetRefusal>;
115
+ evicted: number;
116
+ persisted: boolean | null;
117
+ };
118
+ /** the worker's answer to `spool:write`, `spool:flush`, `spool:writes` and `spool:resolve` */
119
+ export type WriteReply = {
120
+ type: 'spool:write';
121
+ result: WriteQueued | WriteRefusal;
122
+ } | {
123
+ type: 'spool:flush';
124
+ result: ({
125
+ ok: true;
126
+ } & WriteFlush) | WriteRefusal;
127
+ } | {
128
+ type: 'spool:writes';
129
+ result: {
130
+ ok: true;
131
+ writes: WriteStatus[];
132
+ } | WriteRefusal;
133
+ } | {
134
+ type: 'spool:resolve';
135
+ result: WriteResolved | WriteRefusal;
136
+ };
96
137
  /** every reply the worker gives a page */
97
- export type Reply = Hello | Status | ViewId | GraphStatus | LedgerStatus | PartitionStatus | AssetsStatus | HeldStatus;
138
+ export type Reply = WriteReply | Hello | Status | ViewId | DatasetStatus | GraphStatus | LedgerStatus | PartitionStatus | AssetsStatus | HeldStatus;
98
139
  /** the object `serve` wires to service worker events */
99
140
  export type Runtime = {
100
141
  /** a response promise for a request the worker handles, `undefined` for one it leaves alone */