@drupflare/spool 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,278 @@
1
+ import { CONTAMINATION_MS_PER_S, isContaminated, median, quantile, spread } from './derive.js';
2
+ import { attributionalJoules } from './model.js';
3
+ const EMBED_HOSTS = {
4
+ 'www.youtube.com': 'youtube',
5
+ 'youtube.com': 'youtube',
6
+ 'www.youtube-nocookie.com': 'youtube',
7
+ 'youtube-nocookie.com': 'youtube',
8
+ 'player.vimeo.com': 'vimeo'
9
+ };
10
+ /** the bytes a WebP of each image saves: `native` is the file at its own size, `ceiling` the width ceiling, `resized` the rendered width */
11
+ export function imageSavings(rows) {
12
+ let format = 0;
13
+ let resize = 0;
14
+ let lossless = 0;
15
+ for (const row of rows) {
16
+ const arms = row.arms;
17
+ if (!arms?.native)
18
+ continue;
19
+ const ceiling = arms.ceiling?.bytes ?? arms.native.bytes;
20
+ const base = Math.min(row.bytes, ceiling);
21
+ format += row.bytes - base;
22
+ resize += base - Math.min(base, arms.resized?.bytes ?? base);
23
+ if (arms.lossless)
24
+ lossless += Math.max(0, base - arms.lossless.bytes);
25
+ }
26
+ return { format, resize, lossless };
27
+ }
28
+ /** the levers of the image-crawl corpus: the first views of the desktop profile, one entry per page */
29
+ export function crawlLevers(rows) {
30
+ const all = rows;
31
+ const pages = all.filter((r) => r.kind === 'page' &&
32
+ r.profile === 'desktop' &&
33
+ r.variant === 'original' &&
34
+ r.ok === true);
35
+ const images = all.filter((r) => r.kind === 'image' && r.profile === 'desktop');
36
+ const embeds = all.filter((r) => r.kind === 'embed');
37
+ const facade = (provider) => {
38
+ const row = embeds.find((e) => e.provider === provider && e.arm === 'facade');
39
+ return row ? row.bytes + row.docBytes : 0;
40
+ };
41
+ const key = (r) => `${r.site}|${r.page}`;
42
+ const byPage = new Map();
43
+ for (const image of images)
44
+ byPage.set(key(image), [...(byPage.get(key(image)) ?? []), image]);
45
+ const sites = pages.map((p) => p.site);
46
+ const per = (f) => pages.map(f);
47
+ const embedSaved = (page) => page.iframes.reduce((sum, frame) => {
48
+ const provider = EMBED_HOSTS[frame.host];
49
+ return provider ? sum + Math.max(0, frame.bytes - facade(provider)) : sum;
50
+ }, 0);
51
+ const idle = (page) => page.videoTags.length > 0 && page.videoTags.every((t) => !t.autoplay);
52
+ return [
53
+ {
54
+ id: 'images-format',
55
+ status: 'default',
56
+ corpus: 'crawl',
57
+ sites,
58
+ bytes: per((p) => imageSavings(byPage.get(key(p)) ?? []).format)
59
+ },
60
+ {
61
+ id: 'images-resize',
62
+ status: 'default',
63
+ corpus: 'crawl',
64
+ sites,
65
+ bytes: per((p) => imageSavings(byPage.get(key(p)) ?? []).resize)
66
+ },
67
+ {
68
+ id: 'video-preload',
69
+ status: 'default',
70
+ corpus: 'crawl',
71
+ sites,
72
+ bytes: per((p) => (idle(p) ? p.videoBytes : 0))
73
+ },
74
+ {
75
+ id: 'video-defer',
76
+ status: 'option',
77
+ corpus: 'crawl',
78
+ sites,
79
+ bytes: per((p) => (idle(p) ? 0 : p.videoBytes))
80
+ },
81
+ { id: 'embeds', status: 'option', corpus: 'crawl', sites, bytes: per(embedSaved) }
82
+ ];
83
+ }
84
+ /** bytes of image data a visit that never scrolls does not load when `loading="lazy"` is on, by page */
85
+ export function lazyUnscrolled(rows, profile = 'desktop') {
86
+ const pages = rows.filter((r) => r.kind === 'page' && r.profile === profile && r.ok === true);
87
+ const original = new Map(pages.filter((p) => p.variant === 'original').map((p) => [`${p.site}|${p.page}`, p]));
88
+ const pairs = pages
89
+ .filter((p) => p.variant === 'lazy')
90
+ .flatMap((lazy) => {
91
+ const base = original.get(`${lazy.site}|${lazy.page}`);
92
+ return base ? [{ base, lazy }] : [];
93
+ });
94
+ const lcpImages = pairs.filter((p) => p.lazy.lcp?.tag === 'IMG');
95
+ return {
96
+ pairs: pairs.length,
97
+ sites: pairs.map((p) => p.base.site),
98
+ bytes: pairs.map((p) => Math.max(0, p.base.imageBytesLoad - p.lazy.imageBytesLoad)),
99
+ baseLoad: pairs.map((p) => p.base.imageBytesLoad),
100
+ lazyLoad: pairs.map((p) => p.lazy.imageBytesLoad),
101
+ lcpImagePages: lcpImages.length,
102
+ lcpLazyPages: lcpImages.filter((p) => p.lazy.lcp?.loading === 'lazy' && p.base.lcp?.loading !== 'lazy').length
103
+ };
104
+ }
105
+ /** the levers of the trim corpus: the first views of internal pages of the Drupal sites */
106
+ export function trimLevers(trimmed, crawl) {
107
+ const trimSites = trimmed;
108
+ const aggSites = crawl;
109
+ const internal = (sites) => sites.flatMap((s) => s.pages.filter((p) => p.kind === 'internal').map((p) => ({ site: s.site, page: p })));
110
+ const css = internal(trimSites);
111
+ const agg = internal(aggSites).filter((x) => x.page.ok === true && x.page.newAgg);
112
+ return [
113
+ {
114
+ id: 'css-trim',
115
+ status: 'option',
116
+ corpus: 'trim',
117
+ sites: css.map((x) => x.site),
118
+ bytes: css.map((x) => x.page.saved.br5Saved)
119
+ },
120
+ {
121
+ id: 'aggregate-dictionary',
122
+ status: 'max',
123
+ corpus: 'trim',
124
+ sites: agg.map((x) => x.site),
125
+ bytes: agg.map((x) => {
126
+ const a = x.page.newAgg;
127
+ return a.zstdAlone > 0 ? a.transfer * Math.max(0, 1 - a.zstdDict / a.zstdAlone) : 0;
128
+ })
129
+ }
130
+ ];
131
+ }
132
+ /** {@link LeverStats} of a lever's per-page bytes, or null for an empty corpus */
133
+ export function leverStats(sites, bytes) {
134
+ if (bytes.length === 0)
135
+ return null;
136
+ const saving = bytes.filter((b) => b > 0);
137
+ const perSite = new Map();
138
+ bytes.forEach((b, i) => perSite.set(sites[i], [...(perSite.get(sites[i]) ?? []), b]));
139
+ const mean = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length;
140
+ return {
141
+ pages: bytes.length,
142
+ withSaving: saving.length,
143
+ medianPage: quantile(bytes, 0.5),
144
+ medianEligible: saving.length > 0 ? quantile(saving, 0.5) : null,
145
+ medianSite: median([...perSite.values()].map(mean)),
146
+ mean: mean(bytes)
147
+ };
148
+ }
149
+ /** the bytes of several levers added page by page; the levers must come from one corpus and one page order */
150
+ export function addLevers(levers) {
151
+ const first = levers[0];
152
+ if (!first)
153
+ return null;
154
+ return {
155
+ sites: first.sites,
156
+ bytes: first.bytes.map((_, i) => levers.reduce((sum, l) => sum + (l.bytes[i] ?? 0), 0))
157
+ };
158
+ }
159
+ /** the dictionary choices the `dictionary-size` rig measures; see that rig for what each one is */
160
+ export const DICTIONARY_ARMS = [
161
+ 'oracle',
162
+ 'browser',
163
+ 'serverVisit',
164
+ 'serverHeld',
165
+ 'serverOut'
166
+ ];
167
+ /**
168
+ * The savings of each dictionary choice in a `dictionary-size` reading, for `dcz` (zstd) and `dcb`
169
+ * (brotli). A file the choice gives no dictionary, or whose delta is not smaller than brotli 11,
170
+ * saves nothing.
171
+ */
172
+ export function dictionaryArms(rows) {
173
+ const files = rows.filter((r) => r.kind === 'file');
174
+ const stat = (arm, codec) => {
175
+ const saved = (f) => {
176
+ const size = f[arm]?.[codec];
177
+ return size == null ? 0 : Math.max(0, f.br11 - size);
178
+ };
179
+ const share = (group) => {
180
+ const plain = group.reduce((s, f) => s + f.br11, 0);
181
+ return plain > 0 ? group.reduce((s, f) => s + saved(f), 0) / plain : 0;
182
+ };
183
+ const bySite = new Map();
184
+ const byPage = new Map();
185
+ for (const f of files) {
186
+ bySite.set(f.site, [...(bySite.get(f.site) ?? []), f]);
187
+ if (f.pageKind === 'internal') {
188
+ const key = `${f.site}|${f.page}`;
189
+ byPage.set(key, [...(byPage.get(key) ?? []), f]);
190
+ }
191
+ }
192
+ const pages = [...byPage.values()];
193
+ return {
194
+ medianSite: median([...bySite.values()].map(share)),
195
+ pooled: share(files),
196
+ pages: pages.length,
197
+ medianPageBytes: median(pages.map((p) => p.reduce((s, f) => s + saved(f), 0))),
198
+ pagesOver5: pages.filter((p) => share(p) > 0.05).length
199
+ };
200
+ };
201
+ const arms = {};
202
+ for (const arm of DICTIONARY_ARMS)
203
+ arms[arm] = { zstd: stat(arm, 'zstd'), br: stat(arm, 'br') };
204
+ return {
205
+ sites: new Set(files.map((f) => f.site)).size,
206
+ files: files.length,
207
+ br11Bytes: files.reduce((s, f) => s + f.br11, 0),
208
+ arms
209
+ };
210
+ }
211
+ const windowSeconds = (row) => row.elapsedMs / 1000;
212
+ const timed = (rows) => rows.filter((r) => typeof r.elapsedMs === 'number' && r.elapsedMs > 0);
213
+ const tally = (counts, arm) => (counts[String(arm)] ??= { kept: 0, dropped: 0, idleControl: 0 });
214
+ /** kept and dropped windows per arm under the contamination limit; a row with no length (a probe) is not a window */
215
+ export function windowCounts(rows, limit = CONTAMINATION_MS_PER_S) {
216
+ const counts = {};
217
+ for (const row of timed(rows)) {
218
+ tally(counts, row.arm)[isContaminated(row, limit) ? 'dropped' : 'kept']++;
219
+ }
220
+ return counts;
221
+ }
222
+ /** the idle windows of a round may differ by this factor, largest over smallest, before the round is dropped */
223
+ export const IDLE_AGREEMENT = 1.25;
224
+ /**
225
+ * Package watts per arm of an idle-activity reading. A window is dropped when other containers used
226
+ * more than `limit` ms of CPU per second. The rig cannot see a host process or a container that
227
+ * stopped inside a window, so the three idle windows of each round also act as a control: when the
228
+ * largest of them is over {@link IDLE_AGREEMENT} times the smallest, the whole round is dropped. An
229
+ * arm with no kept window is null.
230
+ */
231
+ export function activityFigures(rows, limit = CONTAMINATION_MS_PER_S) {
232
+ const counts = {};
233
+ const watts = (row) => row.pkgJ / windowSeconds(row);
234
+ const calm = timed(rows).filter((row) => {
235
+ const contaminated = isContaminated(row, limit);
236
+ if (contaminated)
237
+ tally(counts, row.arm).dropped++;
238
+ return !contaminated;
239
+ });
240
+ const idleByRound = new Map();
241
+ for (const row of calm) {
242
+ if (row.arm === 'idle') {
243
+ idleByRound.set(row.round, [
244
+ ...(idleByRound.get(row.round) ?? []),
245
+ watts(row)
246
+ ]);
247
+ }
248
+ }
249
+ const noisy = [...idleByRound]
250
+ .filter(([, w]) => w.length > 1 && Math.max(...w) > IDLE_AGREEMENT * Math.min(...w))
251
+ .map(([round]) => round);
252
+ const kept = calm.filter((row) => {
253
+ const drop = noisy.includes(row.round);
254
+ tally(counts, row.arm)[drop ? 'idleControl' : 'kept']++;
255
+ return !drop;
256
+ });
257
+ const idleRound = new Map([...idleByRound].filter(([round]) => !noisy.includes(round)).map(([r, w]) => [r, median(w)]));
258
+ const idleWatts = idleRound.size > 0 ? spread([...idleRound.values()]) : null;
259
+ const arms = {};
260
+ for (const name of new Set(timed(rows).map((row) => String(row.arm)))) {
261
+ if (name === 'idle')
262
+ continue;
263
+ const mine = kept.filter((row) => row.arm === name && idleRound.has(row.round));
264
+ arms[name] =
265
+ mine.length === 0
266
+ ? null
267
+ : {
268
+ windows: mine.length,
269
+ watts: spread(mine.map(watts)),
270
+ extraWatts: spread(mine.map((row) => watts(row) - idleRound.get(row.round)))
271
+ };
272
+ }
273
+ return { idleWatts, noisyRounds: noisy.sort((a, b) => a - b), counts, arms };
274
+ }
275
+ /** attributional joules for `bytes` (the same SWD v4 rate as the headline, no origin term: these are the same requests with fewer bytes) */
276
+ export const ledgerJoules = (bytes) => attributionalJoules(bytes);
277
+ /** megawatt-hours a year when `perView` joules are saved on a share of `views` views */
278
+ export const ledgerMWh = (perView, views, share) => (perView * views * share) / 3.6e9;
@@ -1,6 +1,6 @@
1
1
  import type { Layer } from './inputs.js';
2
2
  /** the rigs that write to `measurements/<rig>/`, one folder each */
3
- export declare const RIGS: readonly ["repeat-nav", "on-wire", "zero-network", "device-cpu", "rapl", "iphone", "payload", "trim"];
3
+ export declare const RIGS: readonly ["repeat-nav", "on-wire", "zero-network", "device-cpu", "rapl", "iphone", "payload", "trim", "link-assets", "image-crawl", "dictionary-size"];
4
4
  /** one of {@link RIGS} */
5
5
  export type Rig = (typeof RIGS)[number];
6
6
  /** the format version {@link parseMeasurement} reads */
@@ -7,7 +7,10 @@ export const RIGS = [
7
7
  'rapl',
8
8
  'iphone',
9
9
  'payload',
10
- 'trim'
10
+ 'trim',
11
+ 'link-assets',
12
+ 'image-crawl',
13
+ 'dictionary-size'
11
14
  ];
12
15
  /** the format version {@link parseMeasurement} reads */
13
16
  export const MEASUREMENT_SCHEMA = 1;
@@ -13,6 +13,10 @@ export type Tag = {
13
13
  name: string;
14
14
  attributes: Attribute[];
15
15
  body?: string;
16
+ /** where the `<` sits in the HTML */
17
+ start: number;
18
+ /** where the closing `>` sits, or the length of the HTML when the tag never closes */
19
+ end: number;
16
20
  };
17
21
  /** the value of the first attribute called `name` on `tag` */
18
22
  export declare const attr: (tag: Tag, name: string) => string | undefined;
@@ -76,7 +76,7 @@ export function tagsOf(html) {
76
76
  }
77
77
  attributes.push({ name, value, start, end: i });
78
78
  }
79
- const tag = { name: found[0].toLowerCase(), attributes };
79
+ const tag = { name: found[0].toLowerCase(), attributes, start: open, end: i };
80
80
  at = Math.min(html.length, i + 1);
81
81
  if (tag.name === 'script' || tag.name === 'style') {
82
82
  const end = new RegExp(`</${tag.name}\\s*>`, 'gi');
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@drupflare/spool",
3
3
  "description": "Client-Side Page Rendering and Execution Framework",
4
- "version": "0.1.0",
4
+ "version": "0.2.0",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
7
7
  "types": "./dist/index.d.ts",
@@ -46,6 +46,10 @@
46
46
  "types": "./dist/report/index.d.ts",
47
47
  "default": "./dist/report/index.js"
48
48
  },
49
+ "./media": {
50
+ "types": "./dist/media/index.d.ts",
51
+ "default": "./dist/media/index.js"
52
+ },
49
53
  "./trim": {
50
54
  "types": "./dist/trim/index.d.ts",
51
55
  "default": "./dist/trim/index.js"
@@ -87,7 +91,7 @@
87
91
  },
88
92
  "devDependencies": {
89
93
  "@azure/functions": "^4.16.5",
90
- "@cloudflare/vitest-pool-workers": "^0.22.0",
94
+ "@cloudflare/vitest-pool-workers": "^0.23.0",
91
95
  "@cloudflare/workers-types": "^5.20260811.1",
92
96
  "@google-cloud/functions-framework": "^5.0.5",
93
97
  "@hapi/hapi": "^21.4.10",
@@ -100,7 +104,7 @@
100
104
  "@types/bun": "^1.4.3",
101
105
  "@types/express": "^5.0.6",
102
106
  "@types/koa": "^3.0.3",
103
- "@types/node": "^24",
107
+ "@types/node": "^26.0.0",
104
108
  "@vitest/coverage-istanbul": "^4.1.0",
105
109
  "elysia": "^1.4.30",
106
110
  "express": "^5.3.0",