@drupflare/spool 0.4.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/README.md +206 -1756
  2. package/dist/angular/index.d.ts +13 -0
  3. package/dist/angular/index.js +25 -0
  4. package/dist/angular/pages.d.ts +28 -0
  5. package/dist/angular/pages.js +85 -0
  6. package/dist/cdn/client.js +1 -1
  7. package/dist/cdn/location.d.ts +6 -0
  8. package/dist/cdn/location.js +8 -0
  9. package/dist/cdn/manifest.json +7 -7
  10. package/dist/cdn/spool-sw.js +1 -1
  11. package/dist/client/bootstrap.d.ts +57 -0
  12. package/dist/client/bootstrap.js +307 -0
  13. package/dist/client/contact.d.ts +24 -0
  14. package/dist/client/contact.js +66 -0
  15. package/dist/client/declared.d.ts +23 -0
  16. package/dist/client/declared.js +68 -0
  17. package/dist/client/entry.js +91 -2
  18. package/dist/client/gate.d.ts +1 -0
  19. package/dist/client/gate.js +148 -0
  20. package/dist/client/index.d.ts +25 -0
  21. package/dist/client/index.js +1 -0
  22. package/dist/client/live.d.ts +66 -0
  23. package/dist/client/live.js +225 -0
  24. package/dist/client/navigate-entry.d.ts +1 -0
  25. package/dist/client/navigate-entry.js +4 -0
  26. package/dist/client/navigate.d.ts +54 -0
  27. package/dist/client/navigate.js +272 -0
  28. package/dist/client/persist.d.ts +2 -0
  29. package/dist/client/persist.js +9 -0
  30. package/dist/client/register.d.ts +11 -1
  31. package/dist/client/register.js +19 -8
  32. package/dist/express/index.d.ts +11 -0
  33. package/dist/express/index.js +209 -0
  34. package/dist/hono/index.d.ts +19 -0
  35. package/dist/hono/index.js +49 -0
  36. package/dist/next/index.d.ts +33 -0
  37. package/dist/next/index.js +41 -0
  38. package/dist/next/pages.d.ts +22 -0
  39. package/dist/next/pages.js +39 -0
  40. package/dist/node/listener.d.ts +3 -0
  41. package/dist/node/listener.js +57 -2
  42. package/dist/nuxt/index.d.ts +22 -0
  43. package/dist/nuxt/index.js +104 -0
  44. package/dist/nuxt/pages.d.ts +26 -0
  45. package/dist/nuxt/pages.js +48 -0
  46. package/dist/nuxt/runtime/index.d.ts +24 -0
  47. package/dist/nuxt/runtime/index.js +58 -0
  48. package/dist/plugins/coverage.d.ts +14 -0
  49. package/dist/plugins/coverage.js +27 -0
  50. package/dist/plugins/mode.d.ts +19 -0
  51. package/dist/plugins/mode.js +20 -0
  52. package/dist/plugins/path.d.ts +16 -0
  53. package/dist/plugins/path.js +139 -0
  54. package/dist/plugins/report.d.ts +22 -0
  55. package/dist/plugins/report.js +18 -0
  56. package/dist/plugins/routes.d.ts +38 -0
  57. package/dist/plugins/routes.js +52 -0
  58. package/dist/plugins/static.d.ts +42 -0
  59. package/dist/plugins/static.js +97 -0
  60. package/dist/protocol/bundle.d.ts +111 -0
  61. package/dist/protocol/bundle.js +428 -0
  62. package/dist/protocol/config.d.ts +65 -1
  63. package/dist/protocol/config.js +28 -0
  64. package/dist/protocol/dataset.d.ts +6 -0
  65. package/dist/protocol/dataset.js +6 -0
  66. package/dist/protocol/envelope.d.ts +33 -1
  67. package/dist/protocol/envelope.js +15 -1
  68. package/dist/protocol/errors.d.ts +1 -1
  69. package/dist/protocol/index-rows.d.ts +22 -0
  70. package/dist/protocol/index-rows.js +53 -0
  71. package/dist/protocol/index.d.ts +2 -0
  72. package/dist/protocol/index.js +2 -0
  73. package/dist/protocol/normalise.d.ts +4 -4
  74. package/dist/protocol/normalise.js +9 -5
  75. package/dist/protocol/ring.d.ts +58 -0
  76. package/dist/protocol/ring.js +6 -0
  77. package/dist/protocol/route.d.ts +57 -11
  78. package/dist/protocol/route.js +95 -3
  79. package/dist/protocol/tag.d.ts +22 -2
  80. package/dist/protocol/tag.js +25 -4
  81. package/dist/protocol/view.d.ts +2 -0
  82. package/dist/protocol/view.js +8 -2
  83. package/dist/protocol/views.d.ts +6 -0
  84. package/dist/protocol/views.js +15 -0
  85. package/dist/protocol/write.d.ts +92 -4
  86. package/dist/protocol/write.js +251 -9
  87. package/dist/proxy/eligible.d.ts +22 -1
  88. package/dist/proxy/eligible.js +62 -15
  89. package/dist/proxy/proxy.js +28 -18
  90. package/dist/react-router/index.d.ts +20 -0
  91. package/dist/react-router/index.js +60 -0
  92. package/dist/react-router/pages.d.ts +34 -0
  93. package/dist/react-router/pages.js +83 -0
  94. package/dist/report/demand.d.ts +49 -0
  95. package/dist/report/demand.js +60 -0
  96. package/dist/report/economics.d.ts +55 -0
  97. package/dist/report/economics.js +189 -0
  98. package/dist/report/headline.d.ts +158 -0
  99. package/dist/report/headline.js +473 -0
  100. package/dist/report/index.d.ts +2 -0
  101. package/dist/report/index.js +2 -0
  102. package/dist/report/inputs.d.ts +79 -0
  103. package/dist/report/inputs.js +115 -0
  104. package/dist/report/measurement.d.ts +1 -1
  105. package/dist/report/measurement.js +5 -1
  106. package/dist/report/traffic.d.ts +8 -0
  107. package/dist/report/traffic.js +7 -0
  108. package/dist/server/bundle.d.ts +55 -0
  109. package/dist/server/bundle.js +172 -0
  110. package/dist/server/dataset.d.ts +8 -2
  111. package/dist/server/dataset.js +51 -4
  112. package/dist/server/embedded.d.ts +7 -3
  113. package/dist/server/embedded.js +7 -3
  114. package/dist/server/index.d.ts +5 -1
  115. package/dist/server/index.js +5 -1
  116. package/dist/server/live-sockets.d.ts +54 -0
  117. package/dist/server/live-sockets.js +131 -0
  118. package/dist/server/live.d.ts +146 -0
  119. package/dist/server/live.js +228 -0
  120. package/dist/server/ring.d.ts +77 -0
  121. package/dist/server/ring.js +311 -0
  122. package/dist/server/spool.d.ts +139 -4
  123. package/dist/server/spool.js +457 -28
  124. package/dist/server/static.d.ts +42 -0
  125. package/dist/server/static.js +29 -0
  126. package/dist/server/stub.d.ts +43 -5
  127. package/dist/server/stub.js +126 -8
  128. package/dist/server/write.d.ts +23 -1
  129. package/dist/server/write.js +96 -0
  130. package/dist/stores/durable-object.d.ts +46 -0
  131. package/dist/stores/durable-object.js +114 -0
  132. package/dist/sw/analytics.d.ts +1 -1
  133. package/dist/sw/analytics.js +2 -2
  134. package/dist/sw/coverage.d.ts +6 -0
  135. package/dist/sw/coverage.js +34 -0
  136. package/dist/sw/family.d.ts +15 -0
  137. package/dist/sw/family.js +48 -0
  138. package/dist/sw/idb.d.ts +2 -0
  139. package/dist/sw/idb.js +2 -2
  140. package/dist/sw/index.d.ts +2 -0
  141. package/dist/sw/index.js +1 -0
  142. package/dist/sw/pack/store.d.ts +128 -0
  143. package/dist/sw/pack/store.js +737 -0
  144. package/dist/sw/policy.d.ts +2 -0
  145. package/dist/sw/policy.js +11 -0
  146. package/dist/sw/probe.d.ts +16 -0
  147. package/dist/sw/probe.js +25 -0
  148. package/dist/sw/router.d.ts +3 -2
  149. package/dist/sw/router.js +15 -2
  150. package/dist/sw/runtime.d.ts +48 -1
  151. package/dist/sw/runtime.js +548 -68
  152. package/dist/sw/scheduler.d.ts +6 -1
  153. package/dist/sw/scheduler.js +22 -5
  154. package/dist/sw/write.d.ts +41 -1
  155. package/dist/sw/write.js +188 -5
  156. package/dist/version.d.ts +1 -1
  157. package/dist/version.js +1 -2
  158. package/dist/vite/index.d.ts +26 -0
  159. package/dist/vite/index.js +66 -0
  160. package/dist/vue/index.d.ts +36 -0
  161. package/dist/vue/index.js +53 -0
  162. package/package.json +64 -4
@@ -0,0 +1,42 @@
1
+ /** where `publishStatic` writes its files: a host folder, an object store or memory */
2
+ export type StaticSink = {
3
+ put(path: string, bytes: Uint8Array, contentType: string): void | Promise<void>;
4
+ /** whether a file is already there; `publishStatic` leaves the immutable ones it finds alone */
5
+ has?(path: string): boolean | Promise<boolean>;
6
+ };
7
+ /** where the stub's site policy came from; `none` means the stub carries no `policy` field */
8
+ export type StaticPolicySource = 'call' | 'option' | 'page' | 'fallback' | 'none';
9
+ /** what `publishStatic` takes beside the sink */
10
+ export type PublishStaticOptions = {
11
+ /** the stub's site policy for this call, ahead of the `policy` option and a learned page; `{}` names no headers */
12
+ policy?: Record<string, string>;
13
+ /** the stub's site policy when the call, the option and a learned page give none */
14
+ fallbackPolicy?: Record<string, string>;
15
+ /** builds the pack at the store cursor first, as `publish(url)` does; ignored without `bundles` */
16
+ url?: URL;
17
+ /**
18
+ * With `cdn.worker`, reads the core's URL before anything is written and throws unless its bytes
19
+ * are this build's core. `true` uses the global `fetch`; pass a function to supply the read. Off
20
+ * by default, so a publish makes no request of its own.
21
+ */
22
+ verifyCore?: boolean | typeof fetch;
23
+ };
24
+ /** what `publishStatic` returns: the store cursor, the paths written and where the policy came from */
25
+ export type PublishStaticResult = {
26
+ version: number;
27
+ paths: string[];
28
+ policy: StaticPolicySource;
29
+ };
30
+ /** a sink that keeps every file in memory, with the paths it was asked to write in `puts` */
31
+ export declare function memoryStaticSink(): StaticSink & {
32
+ files: Map<string, {
33
+ bytes: Uint8Array;
34
+ contentType: string;
35
+ }>;
36
+ puts: string[];
37
+ };
38
+ /**
39
+ * The `_headers` file a static host reads: everything under `/.spool/` is immutable, and the stub
40
+ * lives for `maxAge` seconds, the same lease its own response names.
41
+ */
42
+ export declare function staticHeaders(maxAge: number): string;
@@ -0,0 +1,29 @@
1
+ /** a sink that keeps every file in memory, with the paths it was asked to write in `puts` */
2
+ export function memoryStaticSink() {
3
+ const files = new Map();
4
+ const puts = [];
5
+ return {
6
+ files,
7
+ puts,
8
+ put(path, bytes, contentType) {
9
+ puts.push(path);
10
+ files.set(path, { bytes, contentType });
11
+ },
12
+ has: (path) => files.has(path)
13
+ };
14
+ }
15
+ /**
16
+ * The `_headers` file a static host reads: everything under `/.spool/` is immutable, and the stub
17
+ * lives for `maxAge` seconds, the same lease its own response names.
18
+ */
19
+ export function staticHeaders(maxAge) {
20
+ return [
21
+ '/.spool/*',
22
+ ' Cache-Control: public, max-age=31536000, immutable',
23
+ '',
24
+ '/spool-sw.js',
25
+ ` Cache-Control: max-age=${maxAge}`,
26
+ ' Service-Worker-Allowed: /',
27
+ ''
28
+ ].join('\n');
29
+ }
@@ -1,9 +1,47 @@
1
- import type { SwConfig } from '../protocol/config.js';
1
+ import { type BundleDelta } from '../protocol/bundle.js';
2
+ import type { StubPack, SwConfig } from '../protocol/config.js';
3
+ import type { BundleStore } from './bundle.js';
4
+ /** the most versions back a worker script carries a delta from */
5
+ export declare const STUB_DELTAS = 4;
6
+ /** the most bytes of inline deltas a worker script carries; past it a device fetches one delta */
7
+ export declare const STUB_DELTA_BYTES = 16384;
2
8
  /** the worker script: the config as data, then the shared core or the site's own bundle */
3
9
  export declare function workerScript(config: SwConfig, core?: string): string;
10
+ /** the line that stands in for the core in a stub whose core is served from `url` */
11
+ export declare function workerImport(url: string): string;
4
12
  /**
5
- * The `/spool-sw.js` response. The script lives a day in the browser's cache, which is what lets
6
- * a controlled navigation skip the update check; a deploy changes the bytes and is picked up
7
- * within that bound.
13
+ * Reads `url` and throws unless its bytes hash to `expected`, the `shellHash` of the core the stub
14
+ * names. A script imported by a worker has no integrity check in the browser, so this is the one
15
+ * place the bytes are compared with the build.
8
16
  */
9
- export declare function workerResponse(script: string): Response;
17
+ export declare function checkCore(url: string, expected: string, fetcher: typeof fetch): Promise<void>;
18
+ /**
19
+ * The `/spool-sw.js` response. `script` is awaited while the body streams, so the body can read the
20
+ * store. The script lives for `maxAge` seconds in the browser's cache, which is the lease: a check
21
+ * inside it is answered locally, and a deploy or a new version is picked up after it.
22
+ */
23
+ export declare function workerResponse(script: () => Promise<string>, maxAge: number): Response;
24
+ /** the strong validator of a worker script: the hash of its exact bytes, quoted */
25
+ export declare function workerEtag(text: string): Promise<string>;
26
+ /** whether an `If-None-Match` value names `etag`; weak comparison, so a `W/` copy a proxy made matches */
27
+ export declare function matchesEtag(header: string | null, etag: string): boolean;
28
+ /**
29
+ * The `/spool-sw.js` response to `request`, with an `ETag`. A matching `If-None-Match` gets a `304`
30
+ * with the same headers and no body, so a re-check after the lease costs a few hundred bytes when the
31
+ * script is unchanged. `HEAD` gets the headers alone. The script is read whole, once per version.
32
+ */
33
+ export declare function workerReply(request: Request, script: () => Promise<{
34
+ text: string;
35
+ etag: string;
36
+ }>, maxAge: number): Promise<Response>;
37
+ /** the packs of one store, read once per version; a version's bytes never change */
38
+ export type PackReader = {
39
+ /** the pack a worker script carries for `version`, or undefined when no pack is held for it */
40
+ stub(version: number): Promise<StubPack | undefined>;
41
+ /** the delta from `from` to `to`, or undefined unless both are held and `from` is older */
42
+ delta(from: number, to: number): Promise<BundleDelta | undefined>;
43
+ };
44
+ /** a reader over `bundles`; its memo keeps every pack it has read, so a long-lived site holds them all */
45
+ export declare function packReader(bundles: BundleStore): PackReader;
46
+ /** the response to `GET /.spool/bundle/delta/<from>/<to>`: the delta as JSON, or 404 for a pair not held */
47
+ export declare function respondDelta(reader: PackReader | undefined, path: string): Promise<Response>;
@@ -1,19 +1,137 @@
1
+ import { makeDelta, readBundle } from '../protocol/bundle.js';
2
+ import { shellHash } from '../protocol/hash.js';
1
3
  import { SW_CORE } from './embedded.js';
4
+ import { json } from './sync.js';
5
+ /** the most versions back a worker script carries a delta from */
6
+ export const STUB_DELTAS = 4;
7
+ /** the most bytes of inline deltas a worker script carries; past it a device fetches one delta */
8
+ export const STUB_DELTA_BYTES = 16_384;
9
+ const encoder = new TextEncoder();
2
10
  /** the worker script: the config as data, then the shared core or the site's own bundle */
3
11
  export function workerScript(config, core = SW_CORE) {
4
12
  return `self.__SPOOL_CONFIG__=${JSON.stringify(config)};\n${core}`;
5
13
  }
14
+ /** the line that stands in for the core in a stub whose core is served from `url` */
15
+ export function workerImport(url) {
16
+ return `importScripts(${JSON.stringify(url)});`;
17
+ }
18
+ /**
19
+ * Reads `url` and throws unless its bytes hash to `expected`, the `shellHash` of the core the stub
20
+ * names. A script imported by a worker has no integrity check in the browser, so this is the one
21
+ * place the bytes are compared with the build.
22
+ */
23
+ export async function checkCore(url, expected, fetcher) {
24
+ const response = await fetcher(url, { headers: { 'accept-encoding': 'identity' } });
25
+ if (response.status !== 200) {
26
+ throw new Error(`cannot read the worker core at ${url} (HTTP ${response.status})`);
27
+ }
28
+ const got = await shellHash(await response.text());
29
+ if (got !== expected) {
30
+ throw new Error(`the worker core at ${url} does not match this build (expected ${expected}, got ${got})`);
31
+ }
32
+ }
6
33
  /**
7
- * The `/spool-sw.js` response. The script lives a day in the browser's cache, which is what lets
8
- * a controlled navigation skip the update check; a deploy changes the bytes and is picked up
9
- * within that bound.
34
+ * The `/spool-sw.js` response. `script` is awaited while the body streams, so the body can read the
35
+ * store. The script lives for `maxAge` seconds in the browser's cache, which is the lease: a check
36
+ * inside it is answered locally, and a deploy or a new version is picked up after it.
10
37
  */
11
- export function workerResponse(script) {
12
- return new Response(script, {
38
+ export function workerResponse(script, maxAge) {
39
+ const body = new ReadableStream({
40
+ async start(controller) {
41
+ controller.enqueue(encoder.encode(await script()));
42
+ controller.close();
43
+ }
44
+ });
45
+ return new Response(body, { headers: workerHeaders(maxAge) });
46
+ }
47
+ function workerHeaders(maxAge) {
48
+ return {
49
+ 'content-type': 'text/javascript; charset=utf-8',
50
+ 'cache-control': `max-age=${maxAge}`,
51
+ 'service-worker-allowed': '/'
52
+ };
53
+ }
54
+ /** the strong validator of a worker script: the hash of its exact bytes, quoted */
55
+ export async function workerEtag(text) {
56
+ return `"${await shellHash(text)}"`;
57
+ }
58
+ /** whether an `If-None-Match` value names `etag`; weak comparison, so a `W/` copy a proxy made matches */
59
+ export function matchesEtag(header, etag) {
60
+ if (!header)
61
+ return false;
62
+ const tag = (value) => value.trim().replace(/^W\//, '');
63
+ return header.split(',').some((value) => tag(value) === '*' || tag(value) === tag(etag));
64
+ }
65
+ /**
66
+ * The `/spool-sw.js` response to `request`, with an `ETag`. A matching `If-None-Match` gets a `304`
67
+ * with the same headers and no body, so a re-check after the lease costs a few hundred bytes when the
68
+ * script is unchanged. `HEAD` gets the headers alone. The script is read whole, once per version.
69
+ */
70
+ export async function workerReply(request, script, maxAge) {
71
+ const { text, etag } = await script();
72
+ const headers = { ...workerHeaders(maxAge), etag };
73
+ if (matchesEtag(request.headers.get('if-none-match'), etag)) {
74
+ return new Response(null, { status: 304, headers });
75
+ }
76
+ return new Response(request.method === 'HEAD' ? null : text, { headers });
77
+ }
78
+ /** a reader over `bundles`; its memo keeps every pack it has read, so a long-lived site holds them all */
79
+ export function packReader(bundles) {
80
+ const read = new Map();
81
+ async function packAt(version) {
82
+ if (version < 0)
83
+ return undefined;
84
+ const held = read.get(version);
85
+ if (held)
86
+ return held;
87
+ const bytes = await bundles.get(version);
88
+ if (bytes === undefined)
89
+ return undefined;
90
+ const pack = readBundle(bytes);
91
+ read.set(version, pack);
92
+ return pack;
93
+ }
94
+ return {
95
+ async stub(version) {
96
+ const to = await packAt(version);
97
+ if (!to)
98
+ return undefined;
99
+ const deltas = [];
100
+ let bytes = 0;
101
+ for (let back = 1; back <= STUB_DELTAS; back++) {
102
+ const from = await packAt(version - back);
103
+ if (!from)
104
+ continue;
105
+ const delta = await makeDelta(from.state, to.state);
106
+ const size = encoder.encode(JSON.stringify(delta)).length;
107
+ if (bytes + size > STUB_DELTA_BYTES)
108
+ break;
109
+ bytes += size;
110
+ deltas.push(delta);
111
+ }
112
+ return { version, digest: to.index.digest, deltas };
113
+ },
114
+ async delta(from, to) {
115
+ if (from >= to)
116
+ return undefined;
117
+ const base = await packAt(from);
118
+ const target = await packAt(to);
119
+ if (!base || !target)
120
+ return undefined;
121
+ return makeDelta(base.state, target.state);
122
+ }
123
+ };
124
+ }
125
+ /** the response to `GET /.spool/bundle/delta/<from>/<to>`: the delta as JSON, or 404 for a pair not held */
126
+ export async function respondDelta(reader, path) {
127
+ const match = /^(\d{1,15})\/(\d{1,15})$/.exec(path);
128
+ const delta = reader && match ? await reader.delta(Number(match[1]), Number(match[2])) : undefined;
129
+ if (!delta)
130
+ return json({ error: 'no delta' }, 404);
131
+ return new Response(JSON.stringify(delta), {
13
132
  headers: {
14
- 'content-type': 'text/javascript; charset=utf-8',
15
- 'cache-control': 'max-age=86400',
16
- 'service-worker-allowed': '/'
133
+ 'content-type': 'application/json',
134
+ 'cache-control': 'public, max-age=31536000, immutable'
17
135
  }
18
136
  });
19
137
  }
@@ -1,5 +1,8 @@
1
- import type { Identity } from '../protocol/route.js';
1
+ import type { Dataset } from '../protocol/dataset.js';
2
+ import type { Identity, Route } from '../protocol/route.js';
2
3
  import { type Write } from '../protocol/write.js';
4
+ import type { VersionStore } from '../stores/version-store.js';
5
+ import { type BundleStore } from './bundle.js';
3
6
  /** how many `(class, principal, key)` slots the server remembers an applied write id for */
4
7
  export declare const DEFAULT_REMEMBER = 10000;
5
8
  /** what {@link createWriteServer} takes */
@@ -25,3 +28,22 @@ export type WriteServer = {
25
28
  * reading as a conflict.
26
29
  */
27
30
  export declare function createWriteServer(options: WriteServerOptions): WriteServer;
31
+ /** the most bytes a `Spool-Delta` header may carry; past it the worker fetches the delta route */
32
+ export declare const DELTA_HEADER_BYTES = 16384;
33
+ /** what {@link createResourceWrites} takes */
34
+ export type ResourceWriteOptions = {
35
+ datasets: ReadonlyMap<string, Dataset>;
36
+ store: VersionStore;
37
+ bundles: BundleStore;
38
+ /** the site's routes; their index tables go into every pack a write builds, as `publish` does */
39
+ routes: readonly Route[];
40
+ blockRows?: number;
41
+ maxBytes?: number;
42
+ };
43
+ /**
44
+ * Runs a declared resource write's handler and, when it answers 2xx, attaches the rows the handler
45
+ * changed as `Spool-Delta`. The store moves to the version of the new pack, and that pack is kept, so the
46
+ * delta route and the worker script can reach it. Writes run one at a time, so each starts from the pack
47
+ * the one before it left.
48
+ */
49
+ export declare function createResourceWrites(options: ResourceWriteOptions): (request: Request, next: (request: Request) => Response | Promise<Response>) => Promise<Response>;
@@ -1,5 +1,9 @@
1
+ import { buildPack, bundleDigest, makeDelta, readBundle } from '../protocol/bundle.js';
2
+ import { HEADERS } from '../protocol/envelope.js';
3
+ import { ProtocolError } from '../protocol/errors.js';
1
4
  import { shellHash } from '../protocol/hash.js';
2
5
  import { isWriteId, isWriteKey, readIfMatch, WRITE_HEADERS, WRITE_STATUS } from '../protocol/write.js';
6
+ import { buildBundle, indexTables, memoryBundleStore } from './bundle.js';
3
7
  import { json } from './sync.js';
4
8
  /** how many `(class, principal, key)` slots the server remembers an applied write id for */
5
9
  export const DEFAULT_REMEMBER = 10_000;
@@ -125,3 +129,95 @@ export function createWriteServer(options) {
125
129
  }
126
130
  };
127
131
  }
132
+ /** the most bytes a `Spool-Delta` header may carry; past it the worker fetches the delta route */
133
+ export const DELTA_HEADER_BYTES = 16_384;
134
+ /**
135
+ * Runs a declared resource write's handler and, when it answers 2xx, attaches the rows the handler
136
+ * changed as `Spool-Delta`. The store moves to the version of the new pack, and that pack is kept, so the
137
+ * delta route and the worker script can reach it. Writes run one at a time, so each starts from the pack
138
+ * the one before it left.
139
+ */
140
+ export function createResourceWrites(options) {
141
+ let tail = Promise.resolve();
142
+ return (request, next) => {
143
+ const run = tail.then(() => runResourceWrite(request, next, options));
144
+ tail = run.catch(() => undefined);
145
+ return run;
146
+ };
147
+ }
148
+ /** the rows of one public dataset, read in full; for a dataset whose `changes` answers `reset` */
149
+ async function reseed(dataset, url) {
150
+ const scratch = memoryBundleStore();
151
+ await buildBundle({ store: scratch, datasets: [dataset], version: 0, url });
152
+ const bytes = await scratch.get(0);
153
+ if (!bytes)
154
+ throw new ProtocolError('bad-bundle', `${dataset.name}: the reseed stored no pack`);
155
+ return (await readBundle(bytes)).state.tables[dataset.name] ?? [];
156
+ }
157
+ async function runResourceWrite(request, next, { datasets, store, bundles, routes, blockRows, maxBytes }) {
158
+ const url = new URL(request.url);
159
+ const ctx = { request, url };
160
+ const publics = [...datasets.values()].filter((dataset) => dataset.scope === 'public');
161
+ const before = await store.cursor();
162
+ const heads = new Map();
163
+ for (const dataset of publics)
164
+ heads.set(dataset.name, (await dataset.head(ctx)).cursor);
165
+ if (!(await bundles.get(before))) {
166
+ await buildBundle({
167
+ store: bundles,
168
+ datasets: publics,
169
+ routes,
170
+ version: before,
171
+ url,
172
+ blockRows,
173
+ maxBytes
174
+ });
175
+ }
176
+ const bytes = await bundles.get(before);
177
+ const response = await next(request);
178
+ if (!bytes || !response.ok)
179
+ return response;
180
+ const base = await readBundle(bytes);
181
+ const tables = {};
182
+ for (const dataset of publics)
183
+ tables[dataset.name] = base.state.tables[dataset.name] ?? [];
184
+ const moved = [];
185
+ for (const dataset of publics) {
186
+ const change = await dataset.changes(heads.get(dataset.name), ctx);
187
+ if (change === 'reset') {
188
+ tables[dataset.name] = await reseed(dataset, url);
189
+ moved.push(dataset.name);
190
+ continue;
191
+ }
192
+ if (change.puts.length === 0 && change.deletes.length === 0)
193
+ continue;
194
+ const rows = new Map(tables[dataset.name] ?? []);
195
+ for (const { key } of change.deletes)
196
+ rows.delete(key);
197
+ if (change.puts.length > 0) {
198
+ for (const row of await dataset.read(change.puts, ctx))
199
+ rows.set(dataset.key(row), row);
200
+ }
201
+ tables[dataset.name] = [...rows];
202
+ moved.push(dataset.name);
203
+ }
204
+ if (moved.length === 0)
205
+ return response;
206
+ const to = await store.bump(moved);
207
+ const state = { version: to, tables: { ...tables, ...indexTables(routes, tables) } };
208
+ const built = await buildPack(state, blockRows, maxBytes);
209
+ await bundles.put(to, built.bytes);
210
+ const text = JSON.stringify(await makeDelta(base.state, state));
211
+ const headers = new Headers(response.headers);
212
+ if (new TextEncoder().encode(text).length <= DELTA_HEADER_BYTES) {
213
+ headers.set(HEADERS.delta, text);
214
+ }
215
+ else {
216
+ headers.set(WRITE_HEADERS.digest, await bundleDigest(state.tables));
217
+ }
218
+ return new Response(response.body, {
219
+ status: response.status,
220
+ statusText: response.statusText,
221
+ headers
222
+ });
223
+ }
@@ -1,5 +1,7 @@
1
1
  import { DurableObject } from 'cloudflare:workers';
2
+ import type { LiveDelta } from '../protocol/envelope.js';
2
3
  import type { VectorDiff } from '../protocol/vector.js';
4
+ import { type LiveHub, type LiveStats } from '../server/live.js';
3
5
  import type { VersionStore } from './version-store.js';
4
6
  /**
5
7
  * The version vector in a SQLite-backed Durable Object: one row per tag and one cursor row, so it
@@ -17,3 +19,47 @@ export declare class SpoolVersions extends DurableObject {
17
19
  }
18
20
  /** a {@link VersionStore} over one named instance of the {@link SpoolVersions} namespace */
19
21
  export declare function durableObjectStore(namespace: DurableObjectNamespace<SpoolVersions>, name?: string): VersionStore;
22
+ /**
23
+ * The live streams of every isolate, held by one Durable Object: a page's `/.spool/live` request
24
+ * is forwarded here and its response is the stream, so a bump or a revocation from any isolate
25
+ * reaches every open page. The object stays in memory while a stream is open and is billed for
26
+ * that duration. One instance serves every stream of a site, which is its ceiling. Bind it as a
27
+ * Durable Object class and pass the namespace to {@link durableObjectHub}.
28
+ *
29
+ * A WebSocket upgrade is accepted with the Hibernation API when the hub sets `hibernate`. Those
30
+ * sockets hold no timer, so while no SSE stream is open the object hibernates between a publish, a
31
+ * revoke, a maximum-age alarm and a message that is not a ping.
32
+ */
33
+ export declare class SpoolLive extends DurableObject {
34
+ #private;
35
+ constructor(ctx: DurableObjectState, env: Cloudflare.Env);
36
+ fetch(request: Request): Promise<Response>;
37
+ /** sends a delta to every open stream and socket */
38
+ publish(delta: LiveDelta): void;
39
+ /** sends `revoked` to every stream and socket of `partition` and closes them */
40
+ revoke(partition: string): void;
41
+ webSocketMessage(ws: WebSocket, message: string | ArrayBuffer): void;
42
+ webSocketClose(ws: WebSocket, code: number): void;
43
+ webSocketError(ws: WebSocket): void;
44
+ alarm(): Promise<void>;
45
+ /** what this object has counted since it started */
46
+ stats(): LiveStats;
47
+ }
48
+ /** the options of {@link durableObjectHub} */
49
+ export type DurableObjectHubOptions = {
50
+ /** which instance of the namespace holds the streams; default `spool` */
51
+ name?: string;
52
+ /**
53
+ * Accept a page's WebSocket upgrade with the Hibernation API instead of holding an SSE response.
54
+ * The site's config then tells pages to open a socket, and a page whose upgrade fails falls back
55
+ * to SSE. Default false.
56
+ */
57
+ hibernate?: boolean;
58
+ };
59
+ /**
60
+ * A {@link LiveHub} over one named instance of the {@link SpoolLive} namespace. The hub cannot
61
+ * call back into the app, so a session the app ends is revoked with `spool.revoke`, and one it
62
+ * forgets to revoke is cut off when its stream reaches `maxAge`. The second argument is the
63
+ * instance name or {@link DurableObjectHubOptions}.
64
+ */
65
+ export declare function durableObjectHub(namespace: DurableObjectNamespace<SpoolLive>, options?: string | DurableObjectHubOptions): LiveHub;
@@ -1,4 +1,6 @@
1
1
  import { DurableObject } from 'cloudflare:workers';
2
+ import { decodeOpen, encodeOpen, hibernatingSockets, SOCKET_OPEN_HEADER } from '../server/live-sockets.js';
3
+ import { memoryHub } from '../server/live.js';
2
4
  /**
3
5
  * The version vector in a SQLite-backed Durable Object: one row per tag and one cursor row, so it
4
6
  * survives eviction and restarts. Bind it as a Durable Object class and pass the namespace to
@@ -51,3 +53,115 @@ export function durableObjectStore(namespace, name = 'spool') {
51
53
  diff: (from) => stub().diff(from)
52
54
  };
53
55
  }
56
+ /**
57
+ * The live streams of every isolate, held by one Durable Object: a page's `/.spool/live` request
58
+ * is forwarded here and its response is the stream, so a bump or a revocation from any isolate
59
+ * reaches every open page. The object stays in memory while a stream is open and is billed for
60
+ * that duration. One instance serves every stream of a site, which is its ceiling. Bind it as a
61
+ * Durable Object class and pass the namespace to {@link durableObjectHub}.
62
+ *
63
+ * A WebSocket upgrade is accepted with the Hibernation API when the hub sets `hibernate`. Those
64
+ * sockets hold no timer, so while no SSE stream is open the object hibernates between a publish, a
65
+ * revoke, a maximum-age alarm and a message that is not a ping.
66
+ */
67
+ export class SpoolLive extends DurableObject {
68
+ #hub = memoryHub();
69
+ #sockets = hibernatingSockets({
70
+ pair: () => {
71
+ const pair = new WebSocketPair();
72
+ return [pair[0], pair[1]];
73
+ },
74
+ accept: (server, tags) => this.ctx.acceptWebSocket(server, tags),
75
+ sockets: (tag) => this.ctx.getWebSockets(tag),
76
+ autoResponse: (request, response) => this.ctx.setWebSocketAutoResponse(new WebSocketRequestResponsePair(request, response)),
77
+ alarm: {
78
+ get: () => this.ctx.storage.getAlarm(),
79
+ set: (at) => this.ctx.storage.setAlarm(at)
80
+ }
81
+ });
82
+ constructor(ctx, env) {
83
+ super(ctx, env);
84
+ this.#sockets.restore();
85
+ }
86
+ async fetch(request) {
87
+ if (request.headers.get('upgrade')?.toLowerCase() === 'websocket') {
88
+ const init = decodeOpen(request.headers.get(SOCKET_OPEN_HEADER));
89
+ if (!init)
90
+ return new Response('bad opening request', { status: 400 });
91
+ const client = (await this.#sockets.open(init));
92
+ return new Response(null, { status: 101, webSocket: client });
93
+ }
94
+ const init = (await request.json());
95
+ return this.#hub.open({ ...init, signal: request.signal });
96
+ }
97
+ /** sends a delta to every open stream and socket */
98
+ publish(delta) {
99
+ this.#hub.publish(delta);
100
+ this.#sockets.publish(delta);
101
+ }
102
+ /** sends `revoked` to every stream and socket of `partition` and closes them */
103
+ revoke(partition) {
104
+ this.#hub.revoke(partition);
105
+ this.#sockets.revoke(partition);
106
+ }
107
+ webSocketMessage(ws, message) {
108
+ this.#sockets.message(ws, message);
109
+ }
110
+ webSocketClose(ws, code) {
111
+ // older compatibility dates need the reply; a newer runtime has sent it and ignores this
112
+ try {
113
+ if (code >= 1000 && code < 5000 && ![1004, 1005, 1006, 1015].includes(code))
114
+ ws.close(code);
115
+ else
116
+ ws.close();
117
+ }
118
+ catch {
119
+ // already closed
120
+ }
121
+ }
122
+ webSocketError(ws) {
123
+ try {
124
+ ws.close(1011, 'error');
125
+ }
126
+ catch {
127
+ // already closed
128
+ }
129
+ }
130
+ async alarm() {
131
+ await this.#sockets.alarm();
132
+ }
133
+ /** what this object has counted since it started */
134
+ stats() {
135
+ return this.#hub.stats();
136
+ }
137
+ }
138
+ /**
139
+ * A {@link LiveHub} over one named instance of the {@link SpoolLive} namespace. The hub cannot
140
+ * call back into the app, so a session the app ends is revoked with `spool.revoke`, and one it
141
+ * forgets to revoke is cut off when its stream reaches `maxAge`. The second argument is the
142
+ * instance name or {@link DurableObjectHubOptions}.
143
+ */
144
+ export function durableObjectHub(namespace, options = {}) {
145
+ const { name = 'spool', hibernate = false } = typeof options === 'string' ? { name: options } : options;
146
+ const stub = () => namespace.getByName(name);
147
+ return {
148
+ sockets: hibernate,
149
+ open: (init) => {
150
+ if (hibernate && init.socket) {
151
+ const header = encodeOpen(init);
152
+ if (header === null)
153
+ return new Response('opening frames too large', { status: 413 });
154
+ return stub().fetch(new Request('https://spool.live/socket', {
155
+ headers: { upgrade: 'websocket', [SOCKET_OPEN_HEADER]: header }
156
+ }));
157
+ }
158
+ const { partition, first, leaseMs, keepaliveMs, maxAgeMs } = init;
159
+ return stub().fetch(new Request('https://spool.live/open', {
160
+ method: 'POST',
161
+ body: JSON.stringify({ partition, first, leaseMs, keepaliveMs, maxAgeMs })
162
+ }));
163
+ },
164
+ publish: (delta) => stub().publish(delta),
165
+ revoke: (partition) => stub().revoke(partition)
166
+ };
167
+ }
@@ -36,7 +36,7 @@ export type QueueDeps = {
36
36
  * within {@link CATALOGUE_REASK_MS}. A due ask makes {@link BeaconQueue.flushAtMoment} flush
37
37
  * and nothing else does, so it adds no request on a local view or with nothing held
38
38
  */
39
- catalogue?(): CatalogueAsk | undefined;
39
+ catalogue?(): CatalogueAsk | undefined | Promise<CatalogueAsk | undefined>;
40
40
  };
41
41
  /**
42
42
  * Holds beacon requests on the device. A beacon that reaches the network on its own wakes the
@@ -100,7 +100,7 @@ export class BeaconQueue {
100
100
  const others = held.some((b) => !own(b.event.path));
101
101
  const views = held.filter((b) => b.event.path === VIEW_PATH).length;
102
102
  const wanted = ((await this.#deps.want?.()) ?? []).length > 0;
103
- const asking = (await this.#due(this.#deps.catalogue?.())) !== undefined;
103
+ const asking = (await this.#due(await this.#deps.catalogue?.())) !== undefined;
104
104
  const oldest = Math.min(...held.map((b) => b.event.at));
105
105
  const aged = this.#deps.now() - oldest >= VIEW_HOLD_MS;
106
106
  if (others || wanted || asking || aged || views >= VIEW_BATCH)
@@ -235,7 +235,7 @@ export class BeaconQueue {
235
235
  }
236
236
  const batched = held.filter((h) => inBatch(h.event));
237
237
  let wanted = (await want?.()) ?? [];
238
- let ask = await this.#due(catalogue?.());
238
+ let ask = await this.#due(await catalogue?.());
239
239
  if (batched.length === 0 && wanted.length === 0)
240
240
  return ok;
241
241
  let done = true;
@@ -0,0 +1,6 @@
1
+ import type { SerializedRoute } from '../protocol/route.js';
2
+ /** the GETs a page made that no data family answers, as absolute URLs in first-seen order */
3
+ export declare function coverageReport(requests: readonly {
4
+ method: string;
5
+ url: string;
6
+ }[], families: readonly SerializedRoute[], origin: string): string[];
@@ -0,0 +1,34 @@
1
+ import { matchPattern } from '../protocol/pattern.js';
2
+ /** the GETs a page made that no data family answers, as absolute URLs in first-seen order */
3
+ export function coverageReport(requests, families, origin) {
4
+ const uncovered = new Set();
5
+ for (const { method, url } of requests) {
6
+ if (method !== 'GET')
7
+ continue;
8
+ const parsed = new URL(url);
9
+ if (!families.some((route) => covers(route, parsed, origin)))
10
+ uncovered.add(url);
11
+ }
12
+ return [...uncovered];
13
+ }
14
+ /**
15
+ * A data family covers a GET of its origin (the page's own when it names none) whose path fits its
16
+ * pattern. A query string is covered only by the parameters the family's query takes, the rule
17
+ * `evaluateQuery` applies when the worker reads.
18
+ */
19
+ function covers(route, url, origin) {
20
+ if (route.strategy !== 'data')
21
+ return false;
22
+ if (route.family === undefined && route.origin === undefined)
23
+ return false;
24
+ if ((route.origin ?? origin) !== url.origin)
25
+ return false;
26
+ if (matchPattern(route.pattern, url.pathname) === null)
27
+ return false;
28
+ if (url.search === '')
29
+ return true;
30
+ if (route.query === undefined)
31
+ return false;
32
+ const filter = route.query.filter ?? {};
33
+ return [...url.searchParams.keys()].every((key) => key === 'sort' || key === 'page' || Object.hasOwn(filter, key));
34
+ }