@drupflare/spool 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +1557 -2
  3. package/dist/cdn/client.js +1 -0
  4. package/dist/cdn/location.d.ts +11 -0
  5. package/dist/cdn/location.js +15 -0
  6. package/dist/cdn/manifest.json +16 -0
  7. package/dist/cdn/spool-sw.js +1 -0
  8. package/dist/client/entry.d.ts +1 -0
  9. package/dist/client/entry.js +25 -0
  10. package/dist/client/index.d.ts +3 -0
  11. package/dist/client/index.js +2 -0
  12. package/dist/client/lifecycle.d.ts +10 -0
  13. package/dist/client/lifecycle.js +10 -0
  14. package/dist/client/loads.d.ts +16 -0
  15. package/dist/client/loads.js +41 -0
  16. package/dist/client/register.d.ts +34 -0
  17. package/dist/client/register.js +58 -0
  18. package/dist/index.d.ts +2 -0
  19. package/dist/index.js +2 -0
  20. package/dist/node/listener.d.ts +19 -0
  21. package/dist/node/listener.js +76 -0
  22. package/dist/protocol/artifact.d.ts +54 -0
  23. package/dist/protocol/artifact.js +79 -0
  24. package/dist/protocol/assembly.d.ts +34 -0
  25. package/dist/protocol/assembly.js +72 -0
  26. package/dist/protocol/beacon.d.ts +59 -0
  27. package/dist/protocol/beacon.js +86 -0
  28. package/dist/protocol/collector.d.ts +41 -0
  29. package/dist/protocol/collector.js +84 -0
  30. package/dist/protocol/config.d.ts +89 -0
  31. package/dist/protocol/config.js +13 -0
  32. package/dist/protocol/envelope.d.ts +47 -0
  33. package/dist/protocol/envelope.js +162 -0
  34. package/dist/protocol/errors.d.ts +7 -0
  35. package/dist/protocol/errors.js +9 -0
  36. package/dist/protocol/graph.d.ts +81 -0
  37. package/dist/protocol/graph.js +201 -0
  38. package/dist/protocol/index.d.ts +19 -0
  39. package/dist/protocol/index.js +19 -0
  40. package/dist/protocol/manifest.d.ts +29 -0
  41. package/dist/protocol/manifest.js +94 -0
  42. package/dist/protocol/normalise.d.ts +20 -0
  43. package/dist/protocol/normalise.js +29 -0
  44. package/dist/protocol/pattern.d.ts +12 -0
  45. package/dist/protocol/pattern.js +82 -0
  46. package/dist/protocol/provision.d.ts +100 -0
  47. package/dist/protocol/provision.js +105 -0
  48. package/dist/protocol/route.d.ts +179 -0
  49. package/dist/protocol/route.js +135 -0
  50. package/dist/protocol/sync.d.ts +31 -0
  51. package/dist/protocol/sync.js +87 -0
  52. package/dist/protocol/tag.d.ts +12 -0
  53. package/dist/protocol/tag.js +26 -0
  54. package/dist/protocol/template/compile.d.ts +35 -0
  55. package/dist/protocol/template/compile.js +82 -0
  56. package/dist/protocol/template/interpret.d.ts +9 -0
  57. package/dist/protocol/template/interpret.js +82 -0
  58. package/dist/protocol/vector.d.ts +32 -0
  59. package/dist/protocol/vector.js +56 -0
  60. package/dist/protocol/view.d.ts +110 -0
  61. package/dist/protocol/view.js +201 -0
  62. package/dist/proxy/eligible.d.ts +45 -0
  63. package/dist/proxy/eligible.js +96 -0
  64. package/dist/proxy/index.d.ts +4 -0
  65. package/dist/proxy/index.js +4 -0
  66. package/dist/proxy/proxy.d.ts +86 -0
  67. package/dist/proxy/proxy.js +388 -0
  68. package/dist/proxy/purge.d.ts +10 -0
  69. package/dist/proxy/purge.js +66 -0
  70. package/dist/proxy/tags.d.ts +8 -0
  71. package/dist/proxy/tags.js +36 -0
  72. package/dist/report/derive.d.ts +221 -0
  73. package/dist/report/derive.js +490 -0
  74. package/dist/report/economics.d.ts +14 -0
  75. package/dist/report/economics.js +24 -0
  76. package/dist/report/index.d.ts +8 -0
  77. package/dist/report/index.js +8 -0
  78. package/dist/report/inputs.d.ts +141 -0
  79. package/dist/report/inputs.js +216 -0
  80. package/dist/report/measurement.d.ts +48 -0
  81. package/dist/report/measurement.js +114 -0
  82. package/dist/report/model.d.ts +42 -0
  83. package/dist/report/model.js +74 -0
  84. package/dist/report/thirdparty.d.ts +62 -0
  85. package/dist/report/thirdparty.js +107 -0
  86. package/dist/report/traffic.d.ts +140 -0
  87. package/dist/report/traffic.js +228 -0
  88. package/dist/report/trim.d.ts +81 -0
  89. package/dist/report/trim.js +105 -0
  90. package/dist/server/beacon.d.ts +20 -0
  91. package/dist/server/beacon.js +47 -0
  92. package/dist/server/embedded.d.ts +6 -0
  93. package/dist/server/embedded.js +6 -0
  94. package/dist/server/ga4.d.ts +70 -0
  95. package/dist/server/ga4.js +165 -0
  96. package/dist/server/identity.d.ts +12 -0
  97. package/dist/server/identity.js +17 -0
  98. package/dist/server/index.d.ts +7 -0
  99. package/dist/server/index.js +7 -0
  100. package/dist/server/sink.d.ts +56 -0
  101. package/dist/server/sink.js +133 -0
  102. package/dist/server/size.d.ts +7 -0
  103. package/dist/server/size.js +18 -0
  104. package/dist/server/spool.d.ts +97 -0
  105. package/dist/server/spool.js +463 -0
  106. package/dist/server/stub.d.ts +9 -0
  107. package/dist/server/stub.js +19 -0
  108. package/dist/server/sync.d.ts +12 -0
  109. package/dist/server/sync.js +48 -0
  110. package/dist/server/work.d.ts +22 -0
  111. package/dist/server/work.js +30 -0
  112. package/dist/stores/durable-object.d.ts +19 -0
  113. package/dist/stores/durable-object.js +53 -0
  114. package/dist/stores/index.d.ts +2 -0
  115. package/dist/stores/index.js +2 -0
  116. package/dist/stores/memory.d.ts +4 -0
  117. package/dist/stores/memory.js +10 -0
  118. package/dist/stores/version-store.d.ts +11 -0
  119. package/dist/stores/version-store.js +1 -0
  120. package/dist/sw/analytics.d.ts +81 -0
  121. package/dist/sw/analytics.js +275 -0
  122. package/dist/sw/collect.d.ts +8 -0
  123. package/dist/sw/collect.js +33 -0
  124. package/dist/sw/core.d.ts +1 -0
  125. package/dist/sw/core.js +3 -0
  126. package/dist/sw/freshness.d.ts +15 -0
  127. package/dist/sw/freshness.js +20 -0
  128. package/dist/sw/graph/index.d.ts +67 -0
  129. package/dist/sw/graph/index.js +170 -0
  130. package/dist/sw/graph/invalidate.d.ts +23 -0
  131. package/dist/sw/graph/invalidate.js +109 -0
  132. package/dist/sw/graph/projections.d.ts +7 -0
  133. package/dist/sw/graph/projections.js +9 -0
  134. package/dist/sw/graph/records.d.ts +7 -0
  135. package/dist/sw/graph/records.js +9 -0
  136. package/dist/sw/graph/store.d.ts +32 -0
  137. package/dist/sw/graph/store.js +52 -0
  138. package/dist/sw/idb.d.ts +8 -0
  139. package/dist/sw/idb.js +117 -0
  140. package/dist/sw/index.d.ts +2 -0
  141. package/dist/sw/index.js +1 -0
  142. package/dist/sw/partition.d.ts +21 -0
  143. package/dist/sw/partition.js +137 -0
  144. package/dist/sw/policy.d.ts +8 -0
  145. package/dist/sw/policy.js +24 -0
  146. package/dist/sw/provision.d.ts +39 -0
  147. package/dist/sw/provision.js +61 -0
  148. package/dist/sw/render.d.ts +21 -0
  149. package/dist/sw/render.js +53 -0
  150. package/dist/sw/replica.d.ts +77 -0
  151. package/dist/sw/replica.js +299 -0
  152. package/dist/sw/router.d.ts +42 -0
  153. package/dist/sw/router.js +48 -0
  154. package/dist/sw/routing.d.ts +23 -0
  155. package/dist/sw/routing.js +48 -0
  156. package/dist/sw/runtime.d.ts +112 -0
  157. package/dist/sw/runtime.js +1070 -0
  158. package/dist/sw/scheduler.d.ts +83 -0
  159. package/dist/sw/scheduler.js +149 -0
  160. package/dist/sw/seal.d.ts +37 -0
  161. package/dist/sw/seal.js +183 -0
  162. package/dist/sw/serve.d.ts +15 -0
  163. package/dist/sw/serve.js +63 -0
  164. package/dist/sw/shadow.d.ts +16 -0
  165. package/dist/sw/shadow.js +27 -0
  166. package/dist/sw/static.d.ts +17 -0
  167. package/dist/sw/static.js +81 -0
  168. package/dist/sw/storage.d.ts +64 -0
  169. package/dist/sw/storage.js +34 -0
  170. package/dist/sw/thirdparty.d.ts +17 -0
  171. package/dist/sw/thirdparty.js +36 -0
  172. package/dist/trim/css.d.ts +38 -0
  173. package/dist/trim/css.js +328 -0
  174. package/dist/trim/index.d.ts +3 -0
  175. package/dist/trim/index.js +3 -0
  176. package/dist/trim/manifest.d.ts +67 -0
  177. package/dist/trim/manifest.js +134 -0
  178. package/dist/trim/tokens.d.ts +43 -0
  179. package/dist/trim/tokens.js +142 -0
  180. package/dist/version.d.ts +2 -0
  181. package/dist/version.js +3 -0
  182. package/package.json +146 -5
@@ -0,0 +1,165 @@
1
+ import { GA4_HARD_MS, GA4_RECOMMENDED_MS } from '../protocol/collector.js';
2
+ import { ProtocolError } from '../protocol/errors.js';
3
+ /** most events one Measurement Protocol request may carry */
4
+ export const MP_MAX_EVENTS = 25;
5
+ const NAME = /^[A-Za-z][A-Za-z0-9_]{0,39}$/;
6
+ /** the gtag collect parameters that map to a named event parameter */
7
+ const NAMED = {
8
+ dl: 'page_location',
9
+ dr: 'page_referrer',
10
+ dt: 'page_title',
11
+ ul: 'language',
12
+ sr: 'screen_resolution'
13
+ };
14
+ function paramsOf(text) {
15
+ return new URLSearchParams(text.startsWith('?') ? text.slice(1) : text);
16
+ }
17
+ /**
18
+ * Turns a held gtag collect request into Measurement Protocol events, each stamped with the
19
+ * time it happened. A batched request carries one event per body line and the shared parameters in
20
+ * the query. Events with no client id or an invalid name are skipped. The parameter mapping
21
+ * (`dl`, `dt`, `ep.name`, `epn.name`, `_et`, `sid`) is the one gtag.js uses on the wire; it is read
22
+ * from observed requests, not from a vendor document.
23
+ */
24
+ export function eventsOf(event) {
25
+ const base = paramsOf(event.search ?? '');
26
+ const lines = event.body.split('\n').filter((line) => line.trim() !== '');
27
+ const sets = lines.length > 0 ? lines.map(paramsOf) : [new URLSearchParams()];
28
+ const out = [];
29
+ for (const own of sets) {
30
+ const merged = new URLSearchParams(base);
31
+ for (const [key, value] of own)
32
+ merged.set(key, value);
33
+ const client = merged.get('cid');
34
+ const name = merged.get('en');
35
+ if (!client || !name || !NAME.test(name))
36
+ continue;
37
+ const params = {};
38
+ for (const [key, value] of merged) {
39
+ if (NAMED[key])
40
+ params[NAMED[key]] = value;
41
+ else if (key.startsWith('ep.') && NAME.test(key.slice(3)))
42
+ params[key.slice(3)] = value;
43
+ else if (key.startsWith('epn.') && NAME.test(key.slice(4))) {
44
+ const n = Number(value);
45
+ if (Number.isFinite(n))
46
+ params[key.slice(4)] = n;
47
+ }
48
+ else if (key === '_et' && Number.isFinite(Number(value))) {
49
+ params['engagement_time_msec'] = Number(value);
50
+ }
51
+ else if (key === 'sid' && Number.isFinite(Number(value))) {
52
+ params['session_id'] = Number(value);
53
+ }
54
+ }
55
+ out.push({
56
+ client,
57
+ body: { name, params, timestamp_micros: Math.round(event.at * 1000) }
58
+ });
59
+ }
60
+ return out;
61
+ }
62
+ /**
63
+ * Groups held GA4 events by client, drops those past `maxAgeMs` and splits each client's events
64
+ * into requests of at most {@link MP_MAX_EVENTS}. Order within a client is the order held.
65
+ */
66
+ export function ga4Requests(events, now, maxAgeMs = GA4_RECOMMENDED_MS) {
67
+ const byClient = new Map();
68
+ let dropped = 0;
69
+ for (const event of events) {
70
+ if (event.vendor !== 'ga4')
71
+ continue;
72
+ const made = eventsOf(event);
73
+ if (now - event.at > maxAgeMs) {
74
+ dropped += Math.max(made.length, 1);
75
+ continue;
76
+ }
77
+ for (const { client, body } of made) {
78
+ const list = byClient.get(client) ?? [];
79
+ list.push(body);
80
+ byClient.set(client, list);
81
+ }
82
+ }
83
+ const bodies = [];
84
+ for (const [client_id, list] of byClient) {
85
+ for (let at = 0; at < list.length; at += MP_MAX_EVENTS) {
86
+ bodies.push({ client_id, events: list.slice(at, at + MP_MAX_EVENTS) });
87
+ }
88
+ }
89
+ return { bodies, dropped };
90
+ }
91
+ /**
92
+ * The rules this module checks a body against before sending: a client id, at most 25 events, a
93
+ * valid name on each, and a timestamp no more than 72 hours old and not in the future. It stands
94
+ * in for the validation `/debug/mp/collect` does, which needs a live measurement id and API secret
95
+ * and so is not called from tests.
96
+ */
97
+ export function mpViolations(body, now) {
98
+ const out = [];
99
+ if (!body.client_id)
100
+ out.push('client_id is empty');
101
+ if (body.events.length === 0)
102
+ out.push('no events');
103
+ if (body.events.length > MP_MAX_EVENTS)
104
+ out.push(`more than ${MP_MAX_EVENTS} events`);
105
+ for (const event of body.events) {
106
+ if (!NAME.test(event.name))
107
+ out.push(`bad event name ${event.name}`);
108
+ const age = now - event.timestamp_micros / 1000;
109
+ if (age > GA4_HARD_MS)
110
+ out.push(`${event.name} is older than 72 hours`);
111
+ if (age < -60_000)
112
+ out.push(`${event.name} is in the future`);
113
+ }
114
+ return out;
115
+ }
116
+ /**
117
+ * Forwards held GA4 collect events to the Measurement Protocol with their original timestamps, so a
118
+ * view recorded on a phone offline arrives dated when it happened. Only events of vendor `ga4` are
119
+ * read; the rest of a batch is left to other sinks. A failed request throws, which makes the
120
+ * worker keep the batch and send it again, so a request that did go out before the failure can be
121
+ * sent twice.
122
+ */
123
+ export function ga4Sink(options) {
124
+ const maxAge = options.maxAgeMs ?? GA4_RECOMMENDED_MS;
125
+ if (!(maxAge > 0 && maxAge <= GA4_HARD_MS)) {
126
+ throw new ProtocolError('bad-route', 'ga4 maxAgeMs must be between 1 and 72 hours');
127
+ }
128
+ const post = options.fetch ?? fetch;
129
+ const clock = options.now ?? Date.now;
130
+ const base = `https://www.google-analytics.com${options.debug ? '/debug' : ''}/mp/collect`;
131
+ const url = `${base}?measurement_id=${encodeURIComponent(options.measurementId)}&api_secret=${encodeURIComponent(options.apiSecret)}`;
132
+ let sent = 0;
133
+ let dropped = 0;
134
+ return {
135
+ get sent() {
136
+ return sent;
137
+ },
138
+ get dropped() {
139
+ return dropped;
140
+ },
141
+ async write(events) {
142
+ const plan = ga4Requests(events, clock(), maxAge);
143
+ dropped += plan.dropped;
144
+ for (const body of plan.bodies) {
145
+ const response = await post(url, {
146
+ method: 'POST',
147
+ headers: { 'content-type': 'application/json' },
148
+ body: JSON.stringify(body)
149
+ });
150
+ if (!response.ok)
151
+ throw new Error(`Measurement Protocol answered ${response.status}`);
152
+ sent += body.events.length;
153
+ }
154
+ }
155
+ };
156
+ }
157
+ /** one sink that writes every batch to each of `sinks` in turn */
158
+ export function combineSinks(...sinks) {
159
+ return {
160
+ async write(events) {
161
+ for (const sink of sinks)
162
+ await sink.write(events);
163
+ }
164
+ };
165
+ }
@@ -0,0 +1,12 @@
1
+ import type { Identity } from '../protocol/route.js';
2
+ /** how a request becomes a principal; the secret keys the partition id */
3
+ export type IdentityOptions = {
4
+ resolve(request: Request): Identity | null | Promise<Identity | null>;
5
+ secret: string | Uint8Array;
6
+ };
7
+ /**
8
+ * The `Spool-Partition` value for a principal: an HMAC-SHA-256 under the server secret, so the
9
+ * browser can tell partitions apart without learning the principal, and one principal's id cannot
10
+ * be computed from another's.
11
+ */
12
+ export declare function partitionId(secret: string | Uint8Array, principal: string): Promise<string>;
@@ -0,0 +1,17 @@
1
+ function base64url(bytes) {
2
+ let binary = '';
3
+ for (const byte of bytes)
4
+ binary += String.fromCharCode(byte);
5
+ return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
6
+ }
7
+ /**
8
+ * The `Spool-Partition` value for a principal: an HMAC-SHA-256 under the server secret, so the
9
+ * browser can tell partitions apart without learning the principal, and one principal's id cannot
10
+ * be computed from another's.
11
+ */
12
+ export async function partitionId(secret, principal) {
13
+ const raw = typeof secret === 'string' ? new TextEncoder().encode(secret) : secret;
14
+ const key = await crypto.subtle.importKey('raw', raw, { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']);
15
+ const mac = await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(`spool-partition:1\u0000${principal}`));
16
+ return base64url(new Uint8Array(mac));
17
+ }
@@ -0,0 +1,7 @@
1
+ export { defineRoute } from '../protocol/route.js';
2
+ export * from './ga4.js';
3
+ export * from './identity.js';
4
+ export * from './sink.js';
5
+ export * from './size.js';
6
+ export * from './spool.js';
7
+ export * from './work.js';
@@ -0,0 +1,7 @@
1
+ export { defineRoute } from '../protocol/route.js';
2
+ export * from './ga4.js';
3
+ export * from './identity.js';
4
+ export * from './sink.js';
5
+ export * from './size.js';
6
+ export * from './spool.js';
7
+ export * from './work.js';
@@ -0,0 +1,56 @@
1
+ import type { BeaconEvent } from '../protocol/beacon.js';
2
+ import { type TimedView } from '../protocol/view.js';
3
+ /** where flushed beacon events go: Analytics Engine, a log, a queue */
4
+ export interface Sink {
5
+ write(events: readonly BeaconEvent[]): void | Promise<void>;
6
+ }
7
+ /** a sink that keeps events in `events`, for tests */
8
+ export declare function memorySink(): Sink & {
9
+ events: BeaconEvent[];
10
+ };
11
+ /**
12
+ * The `spool.view` events among `events`, validated, with cross-origin counts joined from the
13
+ * loads reports among them; anything else is skipped. A page reports when it is hidden, which can
14
+ * be a later flush than its view, so pass the whole stream and not one batch.
15
+ */
16
+ export declare function viewsOf(events: readonly BeaconEvent[]): TimedView[];
17
+ /**
18
+ * A sink that writes one JSON line per event through `write`, such as `console.log` or a stream's
19
+ * `write`. A view becomes a flat object (`event`, `at` and the view's fields) and a loads report one
20
+ * (`event`, `view`, `count`, `at`) so a log query can read them; any other request is written as it
21
+ * was held.
22
+ */
23
+ export declare function jsonLinesSink(write: (line: string) => void): Sink;
24
+ /** the part of an Analytics Engine binding the sink calls */
25
+ export interface AnalyticsEngineLike {
26
+ writeDataPoint(point: {
27
+ indexes?: string[];
28
+ blobs?: Array<string | null>;
29
+ doubles?: number[];
30
+ }): void;
31
+ }
32
+ /** longest slice of a body one data point carries, in UTF-8 bytes; a longer body is split across points */
33
+ export declare const MAX_BEACON_BLOB = 4096;
34
+ /**
35
+ * Cuts `text` into slices of at most `limit` UTF-8 bytes, never inside a code point. Text with no
36
+ * bytes still gives one empty slice, so every event leaves one data point at least.
37
+ */
38
+ export declare function splitBytes(text: string, limit?: number): string[];
39
+ /**
40
+ * A sink for a Workers Analytics Engine dataset, one data point per event.
41
+ *
42
+ * | | views | other requests |
43
+ * | --- | --- | --- |
44
+ * | index1 | route pattern | path |
45
+ * | blob1 | `view` | `beacon` |
46
+ * | blob2 to blob9 | tier, reason, engine, scope, strategy, route, id, from | content type, body slice, vendor |
47
+ * | double1 to double6 | at, staleSec, swAgeMs, gapMs, mismatch, crossOriginLoads | at, slice number from 0, slice count |
48
+ *
49
+ * `staleSec`, `gapMs` and `crossOriginLoads` are -1 when unknown; `mismatch` is -1 when nothing was
50
+ * compared, 0 for a match and 1 for a mismatch. A page's loads report is a row with `blob1` of
51
+ * `loads`, the view id as `blob2` and `at` and the count as `double1` and `double2`. Analytics
52
+ * Engine caps an index at 96 bytes and a row at 16 KB, so a body longer than {@link MAX_BEACON_BLOB}
53
+ * bytes is written as several points, one per slice, with the same index, content type, vendor and
54
+ * `at`; join the slices by those and `double2` (the slice number) to rebuild it. Nothing is cut off.
55
+ */
56
+ export declare function analyticsEngineSink(dataset: AnalyticsEngineLike): Sink;
@@ -0,0 +1,133 @@
1
+ import { loadsFromBeacon, viewFromBeacon, viewsWithLoads } from '../protocol/view.js';
2
+ /** a sink that keeps events in `events`, for tests */
3
+ export function memorySink() {
4
+ const events = [];
5
+ return {
6
+ events,
7
+ write(batch) {
8
+ events.push(...batch);
9
+ }
10
+ };
11
+ }
12
+ /**
13
+ * The `spool.view` events among `events`, validated, with cross-origin counts joined from the
14
+ * loads reports among them; anything else is skipped. A page reports when it is hidden, which can
15
+ * be a later flush than its view, so pass the whole stream and not one batch.
16
+ */
17
+ export function viewsOf(events) {
18
+ return viewsWithLoads(events);
19
+ }
20
+ /**
21
+ * A sink that writes one JSON line per event through `write`, such as `console.log` or a stream's
22
+ * `write`. A view becomes a flat object (`event`, `at` and the view's fields) and a loads report one
23
+ * (`event`, `view`, `count`, `at`) so a log query can read them; any other request is written as it
24
+ * was held.
25
+ */
26
+ export function jsonLinesSink(write) {
27
+ return {
28
+ write(batch) {
29
+ for (const event of batch) {
30
+ const view = viewFromBeacon(event);
31
+ const loads = loadsFromBeacon(event);
32
+ if (view)
33
+ write(JSON.stringify({ event: 'spool.view', ...view }));
34
+ else if (loads)
35
+ write(JSON.stringify({ event: 'spool.loads', ...loads }));
36
+ else
37
+ write(JSON.stringify(event));
38
+ }
39
+ }
40
+ };
41
+ }
42
+ /** longest slice of a body one data point carries, in UTF-8 bytes; a longer body is split across points */
43
+ export const MAX_BEACON_BLOB = 4096;
44
+ const encoder = new TextEncoder();
45
+ /**
46
+ * Cuts `text` into slices of at most `limit` UTF-8 bytes, never inside a code point. Text with no
47
+ * bytes still gives one empty slice, so every event leaves one data point at least.
48
+ */
49
+ export function splitBytes(text, limit = MAX_BEACON_BLOB) {
50
+ const parts = [];
51
+ let current = '';
52
+ let size = 0;
53
+ for (const char of text) {
54
+ const n = encoder.encode(char).length;
55
+ if (size + n > limit) {
56
+ parts.push(current);
57
+ current = '';
58
+ size = 0;
59
+ }
60
+ current += char;
61
+ size += n;
62
+ }
63
+ parts.push(current);
64
+ return parts;
65
+ }
66
+ /**
67
+ * A sink for a Workers Analytics Engine dataset, one data point per event.
68
+ *
69
+ * | | views | other requests |
70
+ * | --- | --- | --- |
71
+ * | index1 | route pattern | path |
72
+ * | blob1 | `view` | `beacon` |
73
+ * | blob2 to blob9 | tier, reason, engine, scope, strategy, route, id, from | content type, body slice, vendor |
74
+ * | double1 to double6 | at, staleSec, swAgeMs, gapMs, mismatch, crossOriginLoads | at, slice number from 0, slice count |
75
+ *
76
+ * `staleSec`, `gapMs` and `crossOriginLoads` are -1 when unknown; `mismatch` is -1 when nothing was
77
+ * compared, 0 for a match and 1 for a mismatch. A page's loads report is a row with `blob1` of
78
+ * `loads`, the view id as `blob2` and `at` and the count as `double1` and `double2`. Analytics
79
+ * Engine caps an index at 96 bytes and a row at 16 KB, so a body longer than {@link MAX_BEACON_BLOB}
80
+ * bytes is written as several points, one per slice, with the same index, content type, vendor and
81
+ * `at`; join the slices by those and `double2` (the slice number) to rebuild it. Nothing is cut off.
82
+ */
83
+ export function analyticsEngineSink(dataset) {
84
+ return {
85
+ write(batch) {
86
+ for (const event of batch) {
87
+ const view = viewFromBeacon(event);
88
+ const loads = loadsFromBeacon(event);
89
+ if (loads) {
90
+ dataset.writeDataPoint({
91
+ indexes: [event.path],
92
+ blobs: ['loads', loads.view],
93
+ doubles: [loads.at, loads.count]
94
+ });
95
+ }
96
+ else if (view) {
97
+ dataset.writeDataPoint({
98
+ indexes: [view.route.slice(0, 96)],
99
+ blobs: [
100
+ 'view',
101
+ view.tier,
102
+ view.reason,
103
+ view.engine,
104
+ view.scope,
105
+ view.strategy,
106
+ view.route,
107
+ view.id,
108
+ view.from ?? null
109
+ ],
110
+ doubles: [
111
+ view.at,
112
+ view.staleSec ?? -1,
113
+ view.swAgeMs,
114
+ view.gapMs ?? -1,
115
+ view.mismatch === null ? -1 : Number(view.mismatch),
116
+ view.crossOriginLoads ?? -1
117
+ ]
118
+ });
119
+ }
120
+ else {
121
+ const slices = splitBytes(event.body);
122
+ slices.forEach((slice, n) => {
123
+ dataset.writeDataPoint({
124
+ indexes: [event.path.slice(0, 96)],
125
+ blobs: ['beacon', event.contentType, slice, event.vendor ?? null],
126
+ doubles: [event.at, n, slices.length]
127
+ });
128
+ });
129
+ }
130
+ }
131
+ }
132
+ };
133
+ }
@@ -0,0 +1,7 @@
1
+ /** measures the compressed size of some bytes, in bytes */
2
+ export type Sizer = (bytes: Uint8Array) => number | Promise<number>;
3
+ /**
4
+ * Brotli size through `CompressionStream`, which Node, Bun and Deno have. A runtime that refuses
5
+ * `brotli` gets gzip, which is larger, so it ships fewer entries and never more.
6
+ */
7
+ export declare const compressedSize: Sizer;
@@ -0,0 +1,18 @@
1
+ let format;
2
+ /**
3
+ * Brotli size through `CompressionStream`, which Node, Bun and Deno have. A runtime that refuses
4
+ * `brotli` gets gzip, which is larger, so it ships fewer entries and never more.
5
+ */
6
+ export const compressedSize = async (bytes) => {
7
+ if (format === undefined) {
8
+ try {
9
+ new CompressionStream('brotli');
10
+ format = 'brotli';
11
+ }
12
+ catch {
13
+ format = 'gzip';
14
+ }
15
+ }
16
+ const stream = new Response(bytes).body?.pipeThrough(new CompressionStream(format));
17
+ return (await new Response(stream).arrayBuffer()).byteLength;
18
+ };
@@ -0,0 +1,97 @@
1
+ import { type CollectorConfig } from '../protocol/collector.js';
2
+ import { type AssetMode, type ProvisionConfig, type SwConfig, type ThirdPartyConfig } from '../protocol/config.js';
3
+ import { type Route } from '../protocol/route.js';
4
+ import type { Mode } from '../protocol/view.js';
5
+ import type { VersionStore } from '../stores/version-store.js';
6
+ import { type IdentityOptions } from './identity.js';
7
+ import type { Sink } from './sink.js';
8
+ import { type Sizer } from './size.js';
9
+ import type { WorkSeries } from './work.js';
10
+ /** most changed tags a response carries inline before the worker is left to ask `/.spool/sync` */
11
+ export declare const MAX_PIGGYBACK = 32;
12
+ /** most pages one catalogue request makes the server build, whatever its route suggests */
13
+ export declare const MAX_CATALOGUE_BUILD = 32;
14
+ /** what `createSpool` takes */
15
+ export type SpoolOptions = {
16
+ routes: readonly Route[];
17
+ store: VersionStore;
18
+ identity?: IdentityOptions;
19
+ sink?: Sink;
20
+ /**
21
+ * serve the page runtime from a CDN with SRI; omitted, it is served from
22
+ * `/.spool/<version>/client.js`. `url` defaults to this version's `cdn.drupflare.com` build,
23
+ * whose `integrity` is in the release notes and in `dist/cdn/manifest.json`.
24
+ */
25
+ cdn?: {
26
+ url?: string;
27
+ integrity: string;
28
+ };
29
+ /** same-origin POST paths the worker queues and flushes in one request */
30
+ beacons?: readonly string[];
31
+ /** the mode for routes that do not name one; default `serve`. `shadow` never serves locally */
32
+ mode?: Mode;
33
+ /** the share of views, 0 to 1, that run in shadow while the mode is `serve`; default 0 */
34
+ shadowSample?: number;
35
+ /** whether the worker queues a `spool.view` per page view; default true */
36
+ views?: boolean;
37
+ /** records the time each request took, by route; read it with `work.samples()` */
38
+ work?: WorkSeries;
39
+ /** reads a millisecond clock for `work`; default `performance.now`, which is elapsed time and not CPU */
40
+ meter?: () => number;
41
+ /**
42
+ * priors the worker uses before its own telemetry exists, and the server uses to leave out an
43
+ * artifact whose transfer its expected reuse does not repay
44
+ */
45
+ provision?: ProvisionConfig;
46
+ /** the most compressed bytes of entries one catalogue bundle may carry; default 32 KiB */
47
+ catalogueBytes?: number;
48
+ /** measures an entry's compressed bytes for the catalogue; default Brotli, gzip where the runtime has no Brotli stream */
49
+ catalogueSize?: Sizer;
50
+ /** how the worker treats same-origin subresources; default `cache` */
51
+ assets?: AssetMode;
52
+ /** same-origin paths whose request ends the session; the worker wipes the principal store first */
53
+ logout?: readonly string[];
54
+ /** cross-origin analytics collectors the worker holds and delivers later */
55
+ collectors?: readonly CollectorConfig[];
56
+ /** the opt-in worker cache for versioned cross-origin files */
57
+ thirdParty?: ThirdPartyConfig;
58
+ /**
59
+ * a bundled worker to serve in place of the embedded one, such as a bundle of `serve(config,
60
+ * renderers)` from `@drupflare/spool/sw`; it finds the config in `self.__SPOOL_CONFIG__`
61
+ */
62
+ worker?: string;
63
+ };
64
+ /** per-call overrides for {@link Spool.decorate} */
65
+ export type DecorateOptions = {
66
+ /** the cursor read before the response was built; read now when omitted */
67
+ vector?: number;
68
+ /** `Spool-Clear` tokens to send, such as `['*']` on logout */
69
+ clear?: readonly string[];
70
+ /** also send `Clear-Site-Data: "storage"`, the browser's own wipe of everything the origin holds */
71
+ hardReset?: boolean;
72
+ };
73
+ /** what `createSpool` returns */
74
+ export type Spool = {
75
+ /** answers `/.spool/*` and the worker script, otherwise calls `next` and decorates */
76
+ handle(request: Request, next: (request: Request) => Response | Promise<Response>): Promise<Response>;
77
+ /** a `Response` for `/.spool/*`, `undefined` for anything else */
78
+ respond(request: Request): Promise<Response | undefined>;
79
+ /** adds the envelope headers and the client tag to a response the app produced */
80
+ decorate(request: Request, response: Response, options?: DecorateOptions): Promise<Response>;
81
+ /** bumps the vector after a write; `drop` also removes artifacts by id */
82
+ invalidate(tags: readonly string[], options?: {
83
+ drop?: readonly string[];
84
+ }): Promise<number>;
85
+ /**
86
+ * The `Spool-Partition` value for a principal. A login or account-switch handler sets it on its
87
+ * own response, since the request it answers still carries the old identity: the worker then
88
+ * swaps its store before any local view can read the old one.
89
+ */
90
+ partition(principal: string): Promise<string>;
91
+ /** the `/spool-sw.js` response */
92
+ stub(): Response;
93
+ /** the config embedded in the worker script */
94
+ readonly config: SwConfig;
95
+ };
96
+ /** wires routes, a version store and the endpoints into one handler set */
97
+ export declare function createSpool(options: SpoolOptions): Spool;