@drupflare/spool 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/README.md +499 -43
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/activity.d.ts +37 -0
  6. package/dist/client/activity.js +114 -0
  7. package/dist/client/entry.js +34 -1
  8. package/dist/client/index.d.ts +1 -0
  9. package/dist/client/index.js +1 -0
  10. package/dist/client/register.d.ts +23 -1
  11. package/dist/client/register.js +53 -7
  12. package/dist/media/embeds.d.ts +3 -2
  13. package/dist/media/embeds.js +16 -1
  14. package/dist/protocol/config.d.ts +16 -1
  15. package/dist/protocol/config.js +3 -1
  16. package/dist/protocol/dataset.d.ts +183 -0
  17. package/dist/protocol/dataset.js +206 -0
  18. package/dist/protocol/envelope.d.ts +3 -0
  19. package/dist/protocol/envelope.js +14 -3
  20. package/dist/protocol/errors.d.ts +1 -1
  21. package/dist/protocol/graph.d.ts +23 -4
  22. package/dist/protocol/graph.js +34 -3
  23. package/dist/protocol/hash.d.ts +2 -0
  24. package/dist/protocol/hash.js +8 -0
  25. package/dist/protocol/index.d.ts +7 -0
  26. package/dist/protocol/index.js +7 -0
  27. package/dist/protocol/inline.d.ts +14 -0
  28. package/dist/protocol/inline.js +43 -0
  29. package/dist/protocol/navigation.d.ts +81 -0
  30. package/dist/protocol/navigation.js +249 -0
  31. package/dist/protocol/normalise.d.ts +4 -3
  32. package/dist/protocol/normalise.js +6 -4
  33. package/dist/protocol/plan.d.ts +100 -0
  34. package/dist/protocol/plan.js +661 -0
  35. package/dist/protocol/provision.d.ts +52 -1
  36. package/dist/protocol/provision.js +53 -5
  37. package/dist/protocol/query.d.ts +141 -0
  38. package/dist/protocol/query.js +252 -0
  39. package/dist/protocol/route.d.ts +85 -5
  40. package/dist/protocol/route.js +91 -0
  41. package/dist/protocol/write.d.ts +146 -0
  42. package/dist/protocol/write.js +101 -0
  43. package/dist/proxy/bases.d.ts +53 -0
  44. package/dist/proxy/bases.js +76 -0
  45. package/dist/proxy/index.d.ts +1 -0
  46. package/dist/proxy/index.js +1 -0
  47. package/dist/proxy/proxy.js +8 -2
  48. package/dist/report/capacity.d.ts +59 -0
  49. package/dist/report/capacity.js +76 -0
  50. package/dist/report/channels.d.ts +60 -0
  51. package/dist/report/channels.js +65 -0
  52. package/dist/report/dataset.d.ts +48 -0
  53. package/dist/report/dataset.js +93 -0
  54. package/dist/report/derive.d.ts +285 -0
  55. package/dist/report/derive.js +676 -1
  56. package/dist/report/economics.d.ts +11 -1
  57. package/dist/report/economics.js +19 -1
  58. package/dist/report/index.d.ts +5 -0
  59. package/dist/report/index.js +5 -0
  60. package/dist/report/inputs.d.ts +97 -0
  61. package/dist/report/inputs.js +133 -0
  62. package/dist/report/institution.d.ts +284 -0
  63. package/dist/report/institution.js +225 -0
  64. package/dist/report/ledger.d.ts +28 -0
  65. package/dist/report/ledger.js +30 -0
  66. package/dist/report/loops.d.ts +73 -0
  67. package/dist/report/loops.js +101 -0
  68. package/dist/report/measurement.d.ts +1 -1
  69. package/dist/report/measurement.js +6 -1
  70. package/dist/report/model.d.ts +2 -0
  71. package/dist/report/model.js +4 -0
  72. package/dist/report/thirdparty.d.ts +57 -0
  73. package/dist/report/thirdparty.js +111 -0
  74. package/dist/report/traffic.d.ts +11 -2
  75. package/dist/report/traffic.js +13 -3
  76. package/dist/report/trim.d.ts +56 -0
  77. package/dist/report/trim.js +83 -1
  78. package/dist/server/dataset.d.ts +48 -0
  79. package/dist/server/dataset.js +308 -0
  80. package/dist/server/embedded.d.ts +3 -3
  81. package/dist/server/embedded.js +3 -3
  82. package/dist/server/index.d.ts +3 -0
  83. package/dist/server/index.js +3 -0
  84. package/dist/server/spool.d.ts +40 -0
  85. package/dist/server/spool.js +184 -23
  86. package/dist/server/write.d.ts +27 -0
  87. package/dist/server/write.js +127 -0
  88. package/dist/sw/analytics.d.ts +6 -0
  89. package/dist/sw/analytics.js +14 -0
  90. package/dist/sw/dataset-sync.d.ts +76 -0
  91. package/dist/sw/dataset-sync.js +149 -0
  92. package/dist/sw/dataset.d.ts +91 -0
  93. package/dist/sw/dataset.js +318 -0
  94. package/dist/sw/graph/index.d.ts +31 -1
  95. package/dist/sw/graph/index.js +57 -5
  96. package/dist/sw/graph/invalidate.d.ts +15 -3
  97. package/dist/sw/graph/invalidate.js +42 -4
  98. package/dist/sw/idb.js +78 -9
  99. package/dist/sw/partition.d.ts +6 -1
  100. package/dist/sw/partition.js +21 -2
  101. package/dist/sw/principal-dataset.d.ts +64 -0
  102. package/dist/sw/principal-dataset.js +254 -0
  103. package/dist/sw/render.d.ts +8 -1
  104. package/dist/sw/render.js +28 -1
  105. package/dist/sw/replica.d.ts +14 -3
  106. package/dist/sw/replica.js +46 -5
  107. package/dist/sw/router.js +2 -2
  108. package/dist/sw/runtime.d.ts +43 -2
  109. package/dist/sw/runtime.js +440 -29
  110. package/dist/sw/seal.d.ts +13 -0
  111. package/dist/sw/seal.js +13 -4
  112. package/dist/sw/static.d.ts +18 -0
  113. package/dist/sw/static.js +84 -1
  114. package/dist/sw/storage.d.ts +51 -1
  115. package/dist/sw/storage.js +33 -1
  116. package/dist/sw/write.d.ts +107 -0
  117. package/dist/sw/write.js +442 -0
  118. package/package.json +5 -2
@@ -0,0 +1,114 @@
1
+ /**
2
+ * Lets a site say which of its own loops spool may pause: a loop pauses while its target is off
3
+ * screen, and while the page has been hidden for `hiddenAfterMs`. The hook never touches
4
+ * `setInterval`, `requestAnimationFrame`, `fetch`, audio, WebRTC or any element the site did not
5
+ * hand over.
6
+ */
7
+ export function createActivity(host = browserActivityHost()) {
8
+ return {
9
+ /** pauses the loop while its target is off screen or the page is hidden */
10
+ register(controls, options = {}) {
11
+ const hiddenAfterMs = options.hiddenAfterMs ?? 10_000;
12
+ let onScreen = true;
13
+ let hiddenFor = false;
14
+ let paused = false;
15
+ let done = false;
16
+ let warned = false;
17
+ let timer = null;
18
+ const call = (method) => {
19
+ try {
20
+ controls[method]();
21
+ }
22
+ catch (error) {
23
+ if (!warned) {
24
+ warned = true;
25
+ console.warn('spool.activity', error);
26
+ }
27
+ }
28
+ };
29
+ const sync = () => {
30
+ if (done)
31
+ return;
32
+ const want = !onScreen || hiddenFor;
33
+ if (want === paused)
34
+ return;
35
+ paused = want;
36
+ call(want ? 'pause' : 'resume');
37
+ };
38
+ const clear = () => {
39
+ if (timer === null)
40
+ return;
41
+ host.clearTimeout(timer);
42
+ timer = null;
43
+ };
44
+ const hiddenChanged = () => {
45
+ if (done)
46
+ return;
47
+ if (!host.hidden()) {
48
+ clear();
49
+ hiddenFor = false;
50
+ sync();
51
+ return;
52
+ }
53
+ if (hiddenFor || timer !== null)
54
+ return;
55
+ if (hiddenAfterMs <= 0) {
56
+ hiddenFor = true;
57
+ sync();
58
+ return;
59
+ }
60
+ timer = host.setTimeout(() => {
61
+ timer = null;
62
+ if (host.hidden()) {
63
+ hiddenFor = true;
64
+ sync();
65
+ }
66
+ }, hiddenAfterMs);
67
+ };
68
+ const stopVisibility = host.onVisibilityChange(hiddenChanged);
69
+ const stopObserving = options.target
70
+ ? host.observe(options.target, (next) => {
71
+ onScreen = next;
72
+ sync();
73
+ })
74
+ : () => undefined;
75
+ hiddenChanged();
76
+ return () => {
77
+ if (done)
78
+ return;
79
+ done = true;
80
+ clear();
81
+ stopVisibility();
82
+ stopObserving();
83
+ if (paused) {
84
+ paused = false;
85
+ call('resume');
86
+ }
87
+ };
88
+ }
89
+ };
90
+ }
91
+ export function browserActivityHost() {
92
+ return {
93
+ hidden: () => document.hidden,
94
+ onVisibilityChange(listener) {
95
+ document.addEventListener('visibilitychange', listener);
96
+ return () => document.removeEventListener('visibilitychange', listener);
97
+ },
98
+ observe(target, listener) {
99
+ if (typeof IntersectionObserver === 'undefined') {
100
+ listener(true);
101
+ return () => undefined;
102
+ }
103
+ const observer = new IntersectionObserver((entries) => {
104
+ const entry = entries.at(-1);
105
+ if (entry)
106
+ listener(entry.isIntersecting || entry.intersectionRatio > 0);
107
+ }, { threshold: 0 });
108
+ observer.observe(target);
109
+ return () => observer.disconnect();
110
+ },
111
+ setTimeout: (callback, ms) => setTimeout(callback, ms),
112
+ clearTimeout: (handle) => clearTimeout(handle)
113
+ };
114
+ }
@@ -1,8 +1,41 @@
1
1
  import { CLIENT_TAG_ATTR } from '../protocol/tag.js';
2
+ import { browserActivityHost, createActivity } from './activity.js';
2
3
  import { crossOriginLoads, crossOriginOrigins } from './loads.js';
3
4
  import { register } from './register.js';
4
5
  const ready = register().catch(() => null);
5
- globalThis.spool = { ready };
6
+ const gone = {
7
+ ok: false,
8
+ code: 'unavailable',
9
+ message: 'there is no worker on this page'
10
+ };
11
+ let queued = false;
12
+ /** the write calls, each answered with a refusal when the page has no worker */
13
+ const writes = {
14
+ write: async (...args) => {
15
+ const result = await ready.then((client) => client?.write(...args) ?? gone);
16
+ if (result.ok)
17
+ queued = true;
18
+ return result;
19
+ },
20
+ flush: () => ready.then((client) => client?.flush() ?? gone),
21
+ writes: () => ready.then((client) => client?.writes() ?? gone),
22
+ resolve: (...args) => ready.then((client) => client?.resolve(...args) ?? gone)
23
+ };
24
+ globalThis.spool = { ready, activity: createActivity(browserActivityHost()), ...writes };
25
+ // queued values go out when the page is hidden, which is a moment the browser gives us
26
+ const flushQueued = () => {
27
+ if (queued)
28
+ void writes.flush();
29
+ };
30
+ void ready.then((client) => {
31
+ if (!client)
32
+ return;
33
+ addEventListener('pagehide', flushQueued);
34
+ document.addEventListener('visibilitychange', () => {
35
+ if (document.visibilityState === 'hidden')
36
+ flushQueued();
37
+ });
38
+ });
6
39
  void ready.then(async (client) => {
7
40
  const id = await client?.view().catch(() => null);
8
41
  if (!client || !id)
@@ -1,3 +1,4 @@
1
1
  export type { Hello, Status } from '../sw/runtime.js';
2
+ export * from './activity.js';
2
3
  export * from './lifecycle.js';
3
4
  export * from './register.js';
@@ -1,2 +1,3 @@
1
+ export * from './activity.js';
1
2
  export * from './lifecycle.js';
2
3
  export * from './register.js';
@@ -1,3 +1,4 @@
1
+ import type { WriteFlush, WriteQueued, WriteRefusal, WriteResolved, WriteStatus } from '../protocol/write.js';
1
2
  import type { Hello, Status } from '../sw/runtime.js';
2
3
  import { type Closable } from './lifecycle.js';
3
4
  /** what `register` takes */
@@ -9,8 +10,10 @@ export type RegisterOptions = {
9
10
  helloTimeoutMs?: number;
10
11
  /** the container to register on; default `navigator.serviceWorker` */
11
12
  container?: ServiceWorkerContainer;
12
- /** where `spool:cleared` is dispatched; default `window` */
13
+ /** where `spool:cleared` and `spool:persisted` are dispatched; default `window` */
13
14
  target?: Pick<EventTarget, 'dispatchEvent'>;
15
+ /** the storage manager asked for persistence; default `navigator.storage` */
16
+ storage?: Pick<StorageManager, 'persist' | 'persisted'>;
14
17
  };
15
18
  /** the page's handle on the worker */
16
19
  export type SpoolClient = {
@@ -23,8 +26,27 @@ export type SpoolClient = {
23
26
  view(path?: string): Promise<string | null>;
24
27
  /** tells the worker how many cross-origin resources the page loaded, for view `id`, and from which origins */
25
28
  reportLoads(id: string, count: number, origins?: Record<string, number>): void;
29
+ /** whether the browser granted persistent storage, or null until the worker held a dataset and the page asked */
30
+ persisted(): boolean | null;
26
31
  /** closes `db` when the page is hidden, so the page stays eligible for the back/forward cache */
27
32
  track(db: Closable): () => void;
33
+ /**
34
+ * Queues `value` as the newest for a declared write class and key, edited from `version`. Never
35
+ * throws: it answers a refusal with a `code` while declared writes are off, the class is not
36
+ * declared or no principal is signed in.
37
+ */
38
+ write(name: string, key: string, value: unknown, version: string): Promise<WriteQueued | WriteRefusal>;
39
+ /** sends every queued value now, one request per key; call it before a logout navigation */
40
+ flush(): Promise<({
41
+ ok: true;
42
+ } & WriteFlush) | WriteRefusal>;
43
+ /** the state of each queued value of the signed-in partition, without the values */
44
+ writes(): Promise<{
45
+ ok: true;
46
+ writes: WriteStatus[];
47
+ } | WriteRefusal>;
48
+ /** settles a conflicted key with the local value (re-based and flushed) or the server's */
49
+ resolve(name: string, key: string, use: 'local' | 'server'): Promise<WriteResolved | WriteRefusal>;
28
50
  };
29
51
  /**
30
52
  * Registers the worker with `updateViaCache: 'all'`, so a controlled navigation skips the script
@@ -1,6 +1,8 @@
1
1
  import { PROTOCOL_VERSION } from '../protocol/artifact.js';
2
2
  import { PATHS } from '../protocol/config.js';
3
3
  import { closeOnPageHide } from './lifecycle.js';
4
+ /** how long a flush may take to answer; the requests it sends are the wait */
5
+ const FLUSH_WAIT_MS = 30_000;
4
6
  function ask(worker, message, timeoutMs) {
5
7
  return new Promise((resolve, reject) => {
6
8
  const channel = new MessageChannel();
@@ -26,9 +28,35 @@ export async function register(options = {}) {
26
28
  if (!container)
27
29
  return null;
28
30
  const timeout = options.helloTimeoutMs ?? 5000;
31
+ let granted = null;
32
+ let asked = false;
33
+ let worker;
34
+ // persist() is Window-only, so the worker asks a page once after it first holds a dataset; the
35
+ // grant covers IndexedDB the worker writes. Nothing waits on the answer.
36
+ const persist = async () => {
37
+ if (asked)
38
+ return;
39
+ asked = true;
40
+ const storage = options.storage ?? navigator.storage;
41
+ try {
42
+ granted =
43
+ (await storage?.persisted?.()) === true || (await storage?.persist?.()) === true;
44
+ }
45
+ catch {
46
+ granted = false;
47
+ }
48
+ (options.target ?? window).dispatchEvent(new CustomEvent('spool:persisted', { detail: { granted } }));
49
+ (worker ?? container.controller)?.postMessage({ type: 'spool:persisted', granted });
50
+ };
29
51
  // the worker tells every page when it wipes the principal store, so a page can blank what it shows
30
52
  container.addEventListener?.('message', (event) => {
31
53
  const data = event.data;
54
+ if (data?.type === 'spool:persist')
55
+ void persist();
56
+ if (data?.type === 'spool:written') {
57
+ const { queued, flushed, conflicted } = data;
58
+ (options.target ?? window).dispatchEvent(new CustomEvent('spool:written', { detail: { queued, flushed, conflicted } }));
59
+ }
32
60
  if (data?.type !== 'spool:cleared')
33
61
  return;
34
62
  (options.target ?? window).dispatchEvent(new CustomEvent('spool:cleared', { detail: { reason: data.reason } }));
@@ -38,21 +66,39 @@ export async function register(options = {}) {
38
66
  updateViaCache: 'all'
39
67
  });
40
68
  const registration = await container.ready;
41
- const worker = registration.active;
42
- if (!worker)
69
+ const active = registration.active;
70
+ if (!active)
43
71
  throw new Error('the worker is not active');
44
- const hello = await ask(worker, { type: 'spool:hello', protocol: PROTOCOL_VERSION }, timeout);
72
+ worker = active;
73
+ const hello = await ask(active, { type: 'spool:hello', protocol: PROTOCOL_VERSION }, timeout);
74
+ // a write call never throws: a worker that is gone or silent is a refusal like any other
75
+ const askWrite = async (message, waitMs = timeout) => {
76
+ try {
77
+ const reply = await ask(registration.active ?? active, message, waitMs);
78
+ if (reply?.type === message.type)
79
+ return reply.result;
80
+ }
81
+ catch {
82
+ // falls through to the refusal
83
+ }
84
+ return { ok: false, code: 'unavailable', message: 'the worker did not answer' };
85
+ };
45
86
  return {
46
87
  registration,
47
88
  hello,
48
- status: () => ask(registration.active ?? worker, { type: 'spool:status' }, timeout),
49
- view: async (path = location.pathname) => (await ask(registration.active ?? worker, { type: 'spool:view', path }, timeout)).id,
50
- reportLoads: (id, count, origins) => (registration.active ?? worker).postMessage({
89
+ status: () => ask(registration.active ?? active, { type: 'spool:status' }, timeout),
90
+ view: async (path = location.pathname) => (await ask(registration.active ?? active, { type: 'spool:view', path }, timeout)).id,
91
+ reportLoads: (id, count, origins) => (registration.active ?? active).postMessage({
51
92
  type: 'spool:loads',
52
93
  view: id,
53
94
  count,
54
95
  origins
55
96
  }),
56
- track: (db) => closeOnPageHide(db)
97
+ persisted: () => granted,
98
+ track: (db) => closeOnPageHide(db),
99
+ write: (name, key, value, version) => askWrite({ type: 'spool:write', name, key, value, version }),
100
+ flush: () => askWrite({ type: 'spool:flush' }, Math.max(timeout, FLUSH_WAIT_MS)),
101
+ writes: () => askWrite({ type: 'spool:writes' }),
102
+ resolve: (name, key, use) => askWrite({ type: 'spool:resolve', name, key, use }, Math.max(timeout, FLUSH_WAIT_MS))
57
103
  };
58
104
  }
@@ -19,7 +19,7 @@ export type ResolvedEmbedOptions = {
19
19
  thumbnail: boolean;
20
20
  ignoreCsp: boolean;
21
21
  };
22
- /** what the proxy counted for `embeds`: pages with a rewritten embed, the embeds, and the ones left alone by reason (`csp`, `jsapi`) */
22
+ /** what the proxy counted for `embeds`: pages with a rewritten embed, the embeds, and the ones left alone by reason (`csp`, `jsapi`, `ambient`) */
23
23
  export type EmbedStats = {
24
24
  pages: number;
25
25
  rewritten: number;
@@ -45,7 +45,8 @@ export declare function facadeDocument(embed: {
45
45
  * `http` or protocol-relative `src` on a known embed host and path, with an id of 6 to 64 letters,
46
46
  * digits, `-` and `_`, is touched. An iframe inside an HTML comment or `<noscript>`, one that has
47
47
  * `srcdoc`, and one whose URL sets `enablejsapi=1` (the page's script expects the real player) stay
48
- * as they are; the last is counted as `jsapi`. With `page.csp` set, and no `ignoreCsp`, every
48
+ * as they are; the last is counted as `jsapi`. An embed that plays on its own by design (see
49
+ * `ambient`) stays too and is counted as `ambient`. With `page.csp` set, and no `ignoreCsp`, every
49
50
  * matching iframe is left alone and counted as `csp`. The facade keeps `width`, `height`, `title`,
50
51
  * `class`, `style`, `allow`, `allowfullscreen`, `loading`, `id`, `sandbox` and `referrerpolicy`.
51
52
  * Everything outside the replaced start tags stays byte for byte.
@@ -75,13 +75,24 @@ function facadeTag(html, tag, embed, o) {
75
75
  const doc = escape(facadeDocument(embed, title, o.thumbnail));
76
76
  return `<iframe ${[...kept, `srcdoc="${doc}"`].join(' ')}>`;
77
77
  }
78
+ const AMBIENT_NAME = /(?:^|[^a-z0-9])(?:hero|ambient|background|banner)(?:[^a-z0-9]|$)/i;
79
+ /** a muted loop or a hero background plays on its own by design, so a click-to-load link would change the page */
80
+ function ambient(tag, url) {
81
+ const on = (name) => url.searchParams.get(name) === '1';
82
+ if (on('background'))
83
+ return true;
84
+ if (on('autoplay') && (on('mute') || on('muted') || on('loop')))
85
+ return true;
86
+ return AMBIENT_NAME.test(`${find(tag, 'class')?.value ?? ''} ${find(tag, 'id')?.value ?? ''}`);
87
+ }
78
88
  /**
79
89
  * Replaces each server-rendered YouTube or Vimeo `<iframe src>` in `html` with a facade, as
80
90
  * `options` allow, and says how many it replaced and how many it left alone. Only an `https`,
81
91
  * `http` or protocol-relative `src` on a known embed host and path, with an id of 6 to 64 letters,
82
92
  * digits, `-` and `_`, is touched. An iframe inside an HTML comment or `<noscript>`, one that has
83
93
  * `srcdoc`, and one whose URL sets `enablejsapi=1` (the page's script expects the real player) stay
84
- * as they are; the last is counted as `jsapi`. With `page.csp` set, and no `ignoreCsp`, every
94
+ * as they are; the last is counted as `jsapi`. An embed that plays on its own by design (see
95
+ * `ambient`) stays too and is counted as `ambient`. With `page.csp` set, and no `ignoreCsp`, every
85
96
  * matching iframe is left alone and counted as `csp`. The facade keeps `width`, `height`, `title`,
86
97
  * `class`, `style`, `allow`, `allowfullscreen`, `loading`, `id`, `sandbox` and `referrerpolicy`.
87
98
  * Everything outside the replaced start tags stays byte for byte.
@@ -110,6 +121,10 @@ export function rewriteEmbeds(html, options = {}, page = { csp: false }) {
110
121
  skipped.jsapi = (skipped.jsapi ?? 0) + 1;
111
122
  continue;
112
123
  }
124
+ if (ambient(tag, embed.url)) {
125
+ skipped.ambient = (skipped.ambient ?? 0) + 1;
126
+ continue;
127
+ }
113
128
  if (page.csp && !resolved.ignoreCsp) {
114
129
  skipped.csp = (skipped.csp ?? 0) + 1;
115
130
  continue;
@@ -3,13 +3,16 @@ import type { CollectorConfig } from './collector.js';
3
3
  import type { SerializedRoute } from './route.js';
4
4
  import type { CompiledTemplate } from './template/compile.js';
5
5
  import type { Mode } from './view.js';
6
+ import type { WriteConfig } from './write.js';
6
7
  /** every path under this prefix belongs to spool */
7
8
  export declare const SPOOL_PREFIX = "/.spool/";
8
- /** where the server answers sync, artifact and beacon requests, and where the worker script lives */
9
+ /** where the server answers sync, artifact, dataset and beacon requests, and where the worker script lives */
9
10
  export declare const PATHS: {
10
11
  readonly sync: "/.spool/sync";
11
12
  readonly artifact: "/.spool/artifact";
13
+ readonly dataset: "/.spool/dataset";
12
14
  readonly beacon: "/.spool/beacon";
15
+ readonly write: "/.spool/write";
13
16
  readonly worker: "/spool-sw.js";
14
17
  };
15
18
  /** the same-origin path of the page runtime script for a given version */
@@ -86,4 +89,16 @@ export type SwConfig = {
86
89
  collectors?: CollectorConfig[];
87
90
  /** the opt-in cache for versioned cross-origin files */
88
91
  thirdParty?: ThirdPartyConfig;
92
+ /**
93
+ * The share, above 0 and at most 1, of the storage headroom one dataset may use. The worker
94
+ * refuses a seed whose rows would use more; 1 when absent.
95
+ */
96
+ datasetShare?: number;
97
+ /** the declared write classes; absent while declared writes are off, which is the default */
98
+ writes?: WriteConfig[];
99
+ /** the principal datasets and how long each may answer after its last confirmed sync; absent while the flag is off */
100
+ principalDatasets?: Array<{
101
+ name: string;
102
+ maxStaleMs: number;
103
+ }>;
89
104
  };
@@ -1,10 +1,12 @@
1
1
  /** every path under this prefix belongs to spool */
2
2
  export const SPOOL_PREFIX = '/.spool/';
3
- /** where the server answers sync, artifact and beacon requests, and where the worker script lives */
3
+ /** where the server answers sync, artifact, dataset and beacon requests, and where the worker script lives */
4
4
  export const PATHS = {
5
5
  sync: '/.spool/sync',
6
6
  artifact: '/.spool/artifact',
7
+ dataset: '/.spool/dataset',
7
8
  beacon: '/.spool/beacon',
9
+ write: '/.spool/write',
8
10
  worker: '/spool-sw.js'
9
11
  };
10
12
  /** the same-origin path of the page runtime script for a given version */
@@ -0,0 +1,183 @@
1
+ import { PROTOCOL_VERSION } from './artifact.js';
2
+ import type { Plain, SetProof } from './query.js';
3
+ import type { Identity } from './route.js';
4
+ /** a row of a dataset: a flat or nested JSON object, replaced whole when it changes */
5
+ export type DatasetRow = Plain;
6
+ /** what a dataset's functions receive; a public dataset has no identity */
7
+ export type DatasetContext = {
8
+ /** the request being answered */
9
+ request: Request;
10
+ /** the request URL, parsed */
11
+ url: URL;
12
+ /** who is asking; set for a principal dataset and absent, not null, for a public one */
13
+ principal?: Identity;
14
+ };
15
+ /** what `seed` receives: the context, and the key after which to start when the request pages */
16
+ export type DatasetSeedContext = DatasetContext & {
17
+ /** rows with a key at or before this are not wanted; a backend may use it to skip them */
18
+ after?: string;
19
+ };
20
+ /** a row that left the dataset, and the cursor at which it did */
21
+ export type DatasetDelete = {
22
+ key: string;
23
+ cursor: number;
24
+ };
25
+ /**
26
+ * What `changes` returns: the net change since a cursor. A key appears in `puts` or in `deletes`,
27
+ * never both. A row that entered the dataset's scope with its own version unchanged is a put; a row
28
+ * that left the scope is a delete. Deciding scope membership is `changes`' job, and spool cannot see
29
+ * it. `proof` is the set proof of the whole collection at `cursor`, when the backend keeps one.
30
+ */
31
+ export type DatasetChanges = {
32
+ /** the cursor these changes bring a device to */
33
+ cursor: number;
34
+ /** keys of the rows to send whole; spool reads them with `read` */
35
+ puts: readonly string[];
36
+ /** keys of the rows to remove */
37
+ deletes: readonly DatasetDelete[];
38
+ /** the collection's proof at `cursor`; without it spool reads the whole collection to make one */
39
+ proof?: SetProof;
40
+ };
41
+ /** what `head` returns: the dataset's current cursor and, when the backend keeps one, its proof */
42
+ export type DatasetHead = {
43
+ /** the current cursor; a device that seeds at it asks for changes after it */
44
+ cursor: number;
45
+ /** the proof at `cursor` */
46
+ proof?: SetProof;
47
+ };
48
+ type MaybePromise<T> = T | Promise<T>;
49
+ /** the declaration `defineDataset` takes */
50
+ export type DatasetDeclaration<Row extends DatasetRow = DatasetRow> = {
51
+ /**
52
+ * `public` rows are the same for everyone. A `principal` dataset holds rows that differ per
53
+ * signed-in principal; it needs `maxStaleMs`, `defineDataset`'s `principalDatasets` option and
54
+ * `createSpool`'s `principalDatasets` flag. Unreviewed.
55
+ */
56
+ scope: 'public' | 'principal';
57
+ /**
58
+ * Principal datasets only, and required there: how long held rows may answer a query after the
59
+ * last sync that returned and verified. Past it the query goes to the network. It is also the
60
+ * longest a device keeps serving rows after the server stopped admitting its principal.
61
+ */
62
+ maxStaleMs?: number;
63
+ /** the row's key; unique, and stable while the row exists */
64
+ key(row: Row): string;
65
+ /**
66
+ * Every row in ascending key order (code unit order), as an array or as pages of rows. With
67
+ * `ctx.after` set it may start after that key. It runs once to admit a seed and once per page.
68
+ */
69
+ seed(ctx: DatasetSeedContext): MaybePromise<readonly Row[] | Iterable<readonly Row[]> | AsyncIterable<readonly Row[]>>;
70
+ /** a keyed read: the rows for these keys, omitting any that no longer exist; never a full load */
71
+ read(ids: readonly string[], ctx: DatasetContext): MaybePromise<readonly Row[]>;
72
+ /** the net change since `since`, or `reset` when it cannot answer */
73
+ changes(since: number, ctx: DatasetContext): MaybePromise<DatasetChanges | 'reset'>;
74
+ /** the oldest cursor `changes` can still answer; a device older than this reseeds */
75
+ horizon(ctx: DatasetContext): MaybePromise<number>;
76
+ /** the current cursor, read before a seed so every later change reaches the device */
77
+ head(ctx: DatasetContext): MaybePromise<DatasetHead>;
78
+ /** a version string for the row shape; a device holding another one reseeds */
79
+ schema: string;
80
+ /** most rows a seed may hold */
81
+ maxRows: number;
82
+ /** most bytes of row JSON a seed may hold */
83
+ maxBytes: number;
84
+ /**
85
+ * `always` (the default) answers every seed. `auto` answers a seed only while `datasetBenefit`
86
+ * says the dataset beats both v0.3 and native HTTP; otherwise the seed is refused with `409`.
87
+ */
88
+ provision?: 'always' | 'auto';
89
+ };
90
+ /** a validated dataset */
91
+ export type Dataset<Row extends DatasetRow = DatasetRow> = DatasetDeclaration<Row> & {
92
+ /** the name a device asks for */
93
+ name: string;
94
+ };
95
+ /** what `defineDataset` may be told besides the declaration */
96
+ export type DefineDatasetOptions = {
97
+ /** accept `scope: 'principal'`; `createSpool` still refuses it unless its own flag is on */
98
+ principalDatasets?: boolean;
99
+ };
100
+ /**
101
+ * Declares a dataset. Validates the name, the scope and the limits and returns the declaration with
102
+ * the name attached. Throws `bad-dataset` for a principal scope unless `options.principalDatasets`
103
+ * is true, and for a principal dataset without a positive `maxStaleMs` or with `provision: 'auto'`.
104
+ */
105
+ export declare function defineDataset<Row extends DatasetRow = DatasetRow>(name: string, declaration: DatasetDeclaration<Row>, options?: DefineDatasetOptions): Dataset<Row>;
106
+ /** rows in a seed page when the request names no `limit` */
107
+ export declare const DATASET_PAGE_ROWS = 500;
108
+ /** the most rows one seed page may carry, whatever `limit` asks for */
109
+ export declare const DATASET_PAGE_CAP = 5000;
110
+ /** a row with its key, as it travels */
111
+ export type DatasetPut = {
112
+ key: string;
113
+ row: DatasetRow;
114
+ };
115
+ /**
116
+ * The body of `/.spool/dataset`. A patch has `reset: false` and `proof`. A seed, or a reset, has
117
+ * `reset: true` and one page of rows in `puts`: a page with `next` set is followed by a request
118
+ * with `after=<next>`, and the last page has no `next` and carries `proof`. A page after the first
119
+ * echoes that `after`.
120
+ */
121
+ export type DatasetResponse = {
122
+ protocol: typeof PROTOCOL_VERSION;
123
+ name: string;
124
+ schema: string;
125
+ /** the cursor the response brings a device to */
126
+ cursor: number;
127
+ /** the device drops every row it holds before applying `puts` (unless `after` is set) */
128
+ reset: boolean;
129
+ puts: DatasetPut[];
130
+ deletes: DatasetDelete[];
131
+ /** on a seed page after the first: the key the request paged from */
132
+ after?: string;
133
+ /** on a seed page with more to come: the `after` of the next request */
134
+ next?: string;
135
+ /** the collection's proof at `cursor`; absent on a seed page that has a `next` */
136
+ proof?: SetProof;
137
+ };
138
+ /** what a device asks of `/.spool/dataset` */
139
+ export type DatasetRequest = {
140
+ name: string;
141
+ /** the cursor the device holds; absent for a seed */
142
+ since?: number;
143
+ /** the schema the device holds; required with `since` */
144
+ schema?: string;
145
+ /** the key to page from; seed pages only */
146
+ after?: string;
147
+ /** rows per seed page; the server's default when absent */
148
+ limit?: number;
149
+ };
150
+ /** the query string of a request, for `/.spool/dataset?` */
151
+ export declare function encodeDatasetRequest(request: DatasetRequest): string;
152
+ /** reads a request's parameters; throws `bad-dataset` for a missing name, a bad number, or `since` with `after` */
153
+ export declare function parseDatasetRequest(params: URLSearchParams): DatasetRequest;
154
+ /** validates JSON from `/.spool/dataset`; throws `bad-dataset` */
155
+ export declare function parseDatasetResponse(value: unknown): DatasetResponse;
156
+ /** what {@link applyChanges} returns */
157
+ export type Applied = {
158
+ /** the rows after the response, a new map */
159
+ rows: Map<string, DatasetRow>;
160
+ /** the collection's digest after the response */
161
+ digest: string;
162
+ /** whether the result matches `response.proof`; null when the response carries none */
163
+ matches: boolean | null;
164
+ /** the row hashes after the response; present when the caller passed `options.hashes` */
165
+ hashes?: Map<string, string>;
166
+ };
167
+ /** what {@link applyChanges} may be given besides the rows and the response */
168
+ export type ApplyOptions = {
169
+ /** the hash of every row in `rows`; a replaced or deleted row then costs no hash of its old copy */
170
+ hashes?: ReadonlyMap<string, string>;
171
+ /** the row hash function; {@link rowHash} when absent */
172
+ hash?: (key: string, row: DatasetRow) => Promise<string>;
173
+ };
174
+ /**
175
+ * Applies a response to `rows` and returns the result, leaving `rows` as it was. A reset response
176
+ * without `after` starts from no rows; a later seed page adds to what is held. Deletes apply
177
+ * before puts. `digest` is the digest of `rows` when the caller holds it, which makes the work one
178
+ * row hash per replaced, deleted or added row; without it the digest of `rows` is computed first.
179
+ * With `options.hashes` a replaced or deleted row's old hash is read, not computed, and the result
180
+ * carries the hashes after the response. Pure: no I/O, no clock.
181
+ */
182
+ export declare function applyChanges(rows: ReadonlyMap<string, DatasetRow>, response: DatasetResponse, digest?: string, options?: ApplyOptions): Promise<Applied>;
183
+ export {};