@drupflare/spool 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +193 -1
- package/dist/cdn/manifest.json +1 -1
- package/dist/media/embeds.d.ts +59 -0
- package/dist/media/embeds.js +120 -0
- package/dist/media/headers.d.ts +13 -0
- package/dist/media/headers.js +44 -0
- package/dist/media/html.d.ts +27 -0
- package/dist/media/html.js +93 -0
- package/dist/media/image.d.ts +222 -0
- package/dist/media/image.js +413 -0
- package/dist/media/index.d.ts +6 -0
- package/dist/media/index.js +6 -0
- package/dist/media/lazy.d.ts +26 -0
- package/dist/media/lazy.js +74 -0
- package/dist/media/responsive.d.ts +45 -0
- package/dist/media/responsive.js +98 -0
- package/dist/media/video.d.ts +30 -0
- package/dist/media/video.js +75 -0
- package/dist/proxy/dictionary.d.ts +58 -0
- package/dist/proxy/dictionary.js +197 -0
- package/dist/proxy/index.d.ts +1 -0
- package/dist/proxy/index.js +1 -0
- package/dist/proxy/proxy.d.ts +47 -0
- package/dist/proxy/proxy.js +132 -13
- package/dist/report/derive.d.ts +2 -0
- package/dist/report/derive.js +6 -2
- package/dist/report/index.d.ts +1 -0
- package/dist/report/index.js +1 -0
- package/dist/report/ledger.d.ts +169 -0
- package/dist/report/ledger.js +278 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +4 -1
- package/dist/trim/tokens.d.ts +4 -0
- package/dist/trim/tokens.js +1 -1
- package/package.json +7 -3
package/dist/proxy/proxy.d.ts
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
|
+
import { type EmbedOptions, type EmbedStats } from '../media/embeds.js';
|
|
2
|
+
import { type ImageOptions, type ImageStats } from '../media/image.js';
|
|
3
|
+
import { type VideoOptions } from '../media/video.js';
|
|
1
4
|
import { type AssemblyRoute, type DataRoute } from '../protocol/route.js';
|
|
2
5
|
import { type Spool, type SpoolOptions } from '../server/spool.js';
|
|
3
6
|
import { type TrimManifest } from '../trim/index.js';
|
|
7
|
+
import { type DictionaryOptions, type DictionaryStats } from './dictionary.js';
|
|
4
8
|
/**
|
|
5
9
|
* A route the proxy answers locally, declared like a `createSpool` route minus `load`: the proxy
|
|
6
10
|
* loads the page from the origin. A `data` route parses the origin's JSON. `depends` adds tags to
|
|
@@ -49,6 +53,31 @@ export type ProxyOptions = {
|
|
|
49
53
|
* and every HTML page is checked against the guard first. Without it nothing is trimmed.
|
|
50
54
|
*/
|
|
51
55
|
trim?: TrimManifest;
|
|
56
|
+
/**
|
|
57
|
+
* Converts JPEG and PNG responses to WebP for a browser that accepts it, at the quality and
|
|
58
|
+
* width the request's headers call for, up to a width ceiling of 2560 pixels when the request
|
|
59
|
+
* names none. Identical simultaneous requests share one encoder call, and an encoder that runs
|
|
60
|
+
* past 10 seconds is abandoned for that request. See `ImageOptions`. Without it no image is
|
|
61
|
+
* touched.
|
|
62
|
+
*/
|
|
63
|
+
images?: ImageOptions;
|
|
64
|
+
/**
|
|
65
|
+
* Rewrites `<video>` tags in HTML responses, such as `preload="none"`; see `VideoOptions`.
|
|
66
|
+
* Without it no page is touched for video.
|
|
67
|
+
*/
|
|
68
|
+
video?: VideoOptions;
|
|
69
|
+
/**
|
|
70
|
+
* Replaces server-rendered YouTube and Vimeo `<iframe>` embeds with a link that loads the
|
|
71
|
+
* player on click; see `EmbedOptions`. A page that sends a `Content-Security-Policy` is left
|
|
72
|
+
* alone. Without it no embed is touched.
|
|
73
|
+
*/
|
|
74
|
+
embeds?: EmbedOptions;
|
|
75
|
+
/**
|
|
76
|
+
* Serves a stylesheet or script as `dcz` or `dcb` against an earlier public file of the same
|
|
77
|
+
* directory the visitor already holds (RFC 9842, Chromium 130 and later); see
|
|
78
|
+
* `DictionaryOptions`. Without it no response is marked or encoded.
|
|
79
|
+
*/
|
|
80
|
+
dictionary?: DictionaryOptions;
|
|
52
81
|
};
|
|
53
82
|
/** what the proxy counted since it started: trimmed answers, hash mismatches and guard firings */
|
|
54
83
|
export type TrimStats = {
|
|
@@ -56,6 +85,11 @@ export type TrimStats = {
|
|
|
56
85
|
mismatched: number;
|
|
57
86
|
guarded: number;
|
|
58
87
|
};
|
|
88
|
+
/** what the proxy counted for `video`: pages with at least one rewritten tag, and the tags */
|
|
89
|
+
export type VideoStats = {
|
|
90
|
+
pages: number;
|
|
91
|
+
rewritten: number;
|
|
92
|
+
};
|
|
59
93
|
/** where the proxy writes a guard firing or a hash mismatch to the sink, as JSON `{ kind, ... }` */
|
|
60
94
|
export declare const TRIM_PATH = "/.spool/trim";
|
|
61
95
|
/** the Fetch handler `spoolProxy` returns, with the spool it built */
|
|
@@ -64,6 +98,14 @@ export type SpoolProxy = {
|
|
|
64
98
|
readonly spool: Spool;
|
|
65
99
|
/** live counters; all zero when `trim` is not set */
|
|
66
100
|
readonly trim: Readonly<TrimStats>;
|
|
101
|
+
/** live counters; all zero when `images` is not set */
|
|
102
|
+
readonly images: Readonly<ImageStats>;
|
|
103
|
+
/** live counters; all zero when `video` is not set */
|
|
104
|
+
readonly video: Readonly<VideoStats>;
|
|
105
|
+
/** live counters; all zero when `embeds` is not set */
|
|
106
|
+
readonly embeds: Readonly<EmbedStats>;
|
|
107
|
+
/** live counters; all zero when `dictionary` is not set */
|
|
108
|
+
readonly dictionary: Readonly<DictionaryStats>;
|
|
67
109
|
};
|
|
68
110
|
/**
|
|
69
111
|
* A Fetch handler that sits in front of an origin that cannot load the library, such as PHP or a
|
|
@@ -76,6 +118,11 @@ export type SpoolProxy = {
|
|
|
76
118
|
* The tags the origin sends in `Cache-Tag`, `Surrogate-Key`, `X-Drupal-Cache-Tags` or `xkey`
|
|
77
119
|
* become the page's dependencies, and a page with none depends on its own path.
|
|
78
120
|
*
|
|
121
|
+
* With `images` the proxy answers a JPEG or PNG request from a browser that accepts WebP with the
|
|
122
|
+
* converted file, and adds `Accept-CH` to HTML so later image requests carry their width. With
|
|
123
|
+
* `video` it rewrites `<video>` tags in HTML. Both run on the origin's response before the page is
|
|
124
|
+
* kept, so a held page carries the rewritten tags.
|
|
125
|
+
*
|
|
79
126
|
* Limits: the proxy sees finished HTML, so an artifact is the whole page, the replica holds no
|
|
80
127
|
* records, and two pages never share a fragment. A page is invalidated by `POST /.spool/purge`, or
|
|
81
128
|
* when the proxy sees its body change, and it stays on a device until its route's `maxStale` runs
|
package/dist/proxy/proxy.js
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
import { resolveEmbedOptions, rewriteEmbeds } from '../media/embeds.js';
|
|
2
|
+
import { makePrivate, mergeList, signalsOf } from '../media/headers.js';
|
|
3
|
+
import { convertImage, createImageConverter, IMAGE_HINTS, imageRequest, originSearch } from '../media/image.js';
|
|
4
|
+
import { rewriteLazy } from '../media/lazy.js';
|
|
5
|
+
import { rewriteResponsive } from '../media/responsive.js';
|
|
6
|
+
import { resolveVideoOptions, rewriteVideo } from '../media/video.js';
|
|
1
7
|
import { PATHS } from '../protocol/config.js';
|
|
2
8
|
import { writeEnvelope } from '../protocol/envelope.js';
|
|
3
9
|
import { ProtocolError } from '../protocol/errors.js';
|
|
@@ -6,6 +12,7 @@ import { defineRoute, matchRoute } from '../protocol/route.js';
|
|
|
6
12
|
import { artifactTag } from '../protocol/sync.js';
|
|
7
13
|
import { createSpool } from '../server/spool.js';
|
|
8
14
|
import { guardPage, sourceHash, stripTrimParam, TRIM_PARAM, trimKey } from '../trim/index.js';
|
|
15
|
+
import { createDictionaries } from './dictionary.js';
|
|
9
16
|
import { decline, htmlCharset, isUtf8, setsPrivateCookie } from './eligible.js';
|
|
10
17
|
import { PURGE_PATH, respondPurge } from './purge.js';
|
|
11
18
|
import { pathTag, readTags, stripTags } from './tags.js';
|
|
@@ -43,6 +50,11 @@ const toBytes = (text) => Uint8Array.from(text, (char) => char.charCodeAt(0));
|
|
|
43
50
|
* The tags the origin sends in `Cache-Tag`, `Surrogate-Key`, `X-Drupal-Cache-Tags` or `xkey`
|
|
44
51
|
* become the page's dependencies, and a page with none depends on its own path.
|
|
45
52
|
*
|
|
53
|
+
* With `images` the proxy answers a JPEG or PNG request from a browser that accepts WebP with the
|
|
54
|
+
* converted file, and adds `Accept-CH` to HTML so later image requests carry their width. With
|
|
55
|
+
* `video` it rewrites `<video>` tags in HTML. Both run on the origin's response before the page is
|
|
56
|
+
* kept, so a held page carries the rewritten tags.
|
|
57
|
+
*
|
|
46
58
|
* Limits: the proxy sees finished HTML, so an artifact is the whole page, the replica holds no
|
|
47
59
|
* records, and two pages never share a fragment. A page is invalidated by `POST /.spool/purge`, or
|
|
48
60
|
* when the proxy sees its body change, and it stays on a device until its route's `maxStale` runs
|
|
@@ -74,6 +86,62 @@ export function spoolProxy(options) {
|
|
|
74
86
|
const { store } = options.spool;
|
|
75
87
|
const { trim } = options;
|
|
76
88
|
const stats = { served: 0, mismatched: 0, guarded: 0 };
|
|
89
|
+
const images = options.images ? createImageConverter(options.images) : undefined;
|
|
90
|
+
const video = options.video ? resolveVideoOptions(options.video) : undefined;
|
|
91
|
+
const videoStats = { pages: 0, rewritten: 0 };
|
|
92
|
+
const imageStats = images?.stats ?? {
|
|
93
|
+
converted: 0,
|
|
94
|
+
bytesIn: 0,
|
|
95
|
+
bytesOut: 0,
|
|
96
|
+
cacheHits: 0,
|
|
97
|
+
coalesced: 0,
|
|
98
|
+
lazied: 0,
|
|
99
|
+
responsive: 0,
|
|
100
|
+
skipped: {}
|
|
101
|
+
};
|
|
102
|
+
const dictionaries = options.dictionary
|
|
103
|
+
? createDictionaries(options.dictionary, gate)
|
|
104
|
+
: undefined;
|
|
105
|
+
const embeds = options.embeds ? resolveEmbedOptions(options.embeds) : undefined;
|
|
106
|
+
const embedStats = { pages: 0, rewritten: 0, skipped: {} };
|
|
107
|
+
const lazy = images?.options.lazy || undefined;
|
|
108
|
+
const responsive = images?.options.responsive || undefined;
|
|
109
|
+
const readsHtml = Boolean(trim || video || embeds || lazy || responsive || images?.options.hints);
|
|
110
|
+
/** the embed, responsive and lazy rewrites, which do not depend on the visitor; only the live path counts them */
|
|
111
|
+
function rewriteMarkup(text, headers, pageUrl, count) {
|
|
112
|
+
let out = text;
|
|
113
|
+
if (embeds) {
|
|
114
|
+
const result = rewriteEmbeds(out, embeds, {
|
|
115
|
+
csp: headers.has('content-security-policy')
|
|
116
|
+
});
|
|
117
|
+
out = result.html;
|
|
118
|
+
if (count) {
|
|
119
|
+
if (result.rewritten > 0) {
|
|
120
|
+
embedStats.pages++;
|
|
121
|
+
embedStats.rewritten += result.rewritten;
|
|
122
|
+
}
|
|
123
|
+
for (const [why, n] of Object.entries(result.skipped)) {
|
|
124
|
+
embedStats.skipped[why] = (embedStats.skipped[why] ?? 0) + n;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
if (responsive && images) {
|
|
129
|
+
const result = rewriteResponsive(out, responsive, {
|
|
130
|
+
pageUrl,
|
|
131
|
+
paths: images.options.paths
|
|
132
|
+
});
|
|
133
|
+
out = result.html;
|
|
134
|
+
if (count)
|
|
135
|
+
imageStats.responsive += result.rewritten;
|
|
136
|
+
}
|
|
137
|
+
if (lazy) {
|
|
138
|
+
const result = rewriteLazy(out, lazy);
|
|
139
|
+
out = result.html;
|
|
140
|
+
if (count)
|
|
141
|
+
imageStats.lazied += result.rewritten;
|
|
142
|
+
}
|
|
143
|
+
return out;
|
|
144
|
+
}
|
|
77
145
|
function note(kind, detail) {
|
|
78
146
|
if (!options.spool.sink)
|
|
79
147
|
return;
|
|
@@ -126,11 +194,18 @@ export function spoolProxy(options) {
|
|
|
126
194
|
headers.set('x-forwarded-proto', publicUrl.protocol.slice(0, -1));
|
|
127
195
|
return headers;
|
|
128
196
|
}
|
|
129
|
-
function forward(request, url) {
|
|
197
|
+
function forward(request, url, whole) {
|
|
130
198
|
const read = request.method === 'GET' || request.method === 'HEAD';
|
|
131
|
-
|
|
199
|
+
const headers = originHeaders(request.headers, url);
|
|
200
|
+
// a conversion needs the whole file, so the origin gets no conditional request
|
|
201
|
+
if (whole) {
|
|
202
|
+
headers.delete('if-none-match');
|
|
203
|
+
headers.delete('if-modified-since');
|
|
204
|
+
}
|
|
205
|
+
const search = images ? originSearch(images, url) : url.search;
|
|
206
|
+
return new Request(new URL(url.pathname + stripTrimParam(search), base), {
|
|
132
207
|
method: request.method,
|
|
133
|
-
headers
|
|
208
|
+
headers,
|
|
134
209
|
redirect: 'manual',
|
|
135
210
|
signal: request.signal,
|
|
136
211
|
...(read || !request.body ? {} : { body: request.body, duplex: 'half' })
|
|
@@ -204,7 +279,13 @@ export function spoolProxy(options) {
|
|
|
204
279
|
if (!data && !isUtf8(htmlCharset(response.headers, bytes))) {
|
|
205
280
|
throw new ProtocolError('bad-artifact', `${path} is not kept (charset)`);
|
|
206
281
|
}
|
|
207
|
-
|
|
282
|
+
let text = guarded(new TextDecoder().decode(bytes), ctx.url.href);
|
|
283
|
+
// a signal-triggered rewrite is never part of a held copy
|
|
284
|
+
if (!data && video?.when === 'always') {
|
|
285
|
+
text = rewriteVideo(text, video, { saveData: false, slow: false }).html;
|
|
286
|
+
}
|
|
287
|
+
if (!data)
|
|
288
|
+
text = rewriteMarkup(text, response.headers, ctx.url.href, false);
|
|
208
289
|
if (text.length > gate.maxBytes) {
|
|
209
290
|
throw new ProtocolError('bad-artifact', `${path} is not kept (size)`);
|
|
210
291
|
}
|
|
@@ -289,9 +370,10 @@ export function spoolProxy(options) {
|
|
|
289
370
|
return answered;
|
|
290
371
|
if (url.pathname === PATHS.worker && request.method === 'GET')
|
|
291
372
|
return spool.stub();
|
|
373
|
+
const converting = images !== undefined && imageRequest(images, request);
|
|
292
374
|
let upstream;
|
|
293
375
|
try {
|
|
294
|
-
upstream = await call(forward(request, url));
|
|
376
|
+
upstream = await call(forward(request, url, converting));
|
|
295
377
|
}
|
|
296
378
|
catch {
|
|
297
379
|
return new Response('bad gateway', {
|
|
@@ -299,10 +381,13 @@ export function spoolProxy(options) {
|
|
|
299
381
|
headers: { 'content-type': 'text/plain', 'cache-control': 'no-store' }
|
|
300
382
|
});
|
|
301
383
|
}
|
|
302
|
-
if (
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
384
|
+
if (images && converting)
|
|
385
|
+
upstream = await convertImage(images, request, upstream);
|
|
386
|
+
if (request.method === 'GET') {
|
|
387
|
+
const key = trim ? trimKey(url.href) : '';
|
|
388
|
+
const entry = trim?.styles[key];
|
|
389
|
+
if (trim &&
|
|
390
|
+
entry &&
|
|
306
391
|
url.searchParams.get(TRIM_PARAM) !== '0' &&
|
|
307
392
|
upstream.status === 200 &&
|
|
308
393
|
(upstream.headers.get('content-type') ?? '').toLowerCase().includes('text/css')) {
|
|
@@ -324,14 +409,32 @@ export function spoolProxy(options) {
|
|
|
324
409
|
headers
|
|
325
410
|
});
|
|
326
411
|
}
|
|
327
|
-
else if (
|
|
412
|
+
else if (readsHtml &&
|
|
413
|
+
isHtml(upstream) &&
|
|
414
|
+
![204, 205, 304].includes(upstream.status)) {
|
|
328
415
|
const bytes = new Uint8Array(await upstream.arrayBuffer());
|
|
329
|
-
// another charset is rewritten byte for byte; the
|
|
416
|
+
// another charset is rewritten byte for byte; the edits insert ASCII only
|
|
330
417
|
const utf8 = isUtf8(htmlCharset(upstream.headers, bytes));
|
|
331
418
|
const text = utf8 ? new TextDecoder().decode(bytes) : fromBytes(bytes);
|
|
332
|
-
|
|
419
|
+
let verdict = guarded(text, url.href);
|
|
333
420
|
const headers = new Headers(upstream.headers);
|
|
334
421
|
headers.delete('content-length');
|
|
422
|
+
if (video) {
|
|
423
|
+
const result = rewriteVideo(verdict, video, signalsOf(request.headers));
|
|
424
|
+
if (result.rewritten > 0) {
|
|
425
|
+
verdict = result.html;
|
|
426
|
+
videoStats.pages++;
|
|
427
|
+
videoStats.rewritten += result.rewritten;
|
|
428
|
+
// a page rewritten for one visitor's connection is not another's copy
|
|
429
|
+
if (video.when !== 'always') {
|
|
430
|
+
makePrivate(headers);
|
|
431
|
+
mergeList(headers, 'vary', ['Save-Data', 'ECT']);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
verdict = rewriteMarkup(verdict, upstream.headers, url.href, true);
|
|
436
|
+
if (images?.options.hints)
|
|
437
|
+
mergeList(headers, 'accept-ch', IMAGE_HINTS);
|
|
335
438
|
if (verdict !== text)
|
|
336
439
|
weaken(headers);
|
|
337
440
|
const body = verdict === text ? bytes : utf8 ? verdict : toBytes(verdict);
|
|
@@ -351,6 +454,9 @@ export function spoolProxy(options) {
|
|
|
351
454
|
if (request.method !== 'GET' && request.method !== 'HEAD') {
|
|
352
455
|
return new Response(upstream.body, init);
|
|
353
456
|
}
|
|
457
|
+
if (dictionaries?.applies(request, upstream)) {
|
|
458
|
+
return dictionaries.apply(request, new Response(upstream.body, init));
|
|
459
|
+
}
|
|
354
460
|
const hit = matchRoute(routes, url.pathname);
|
|
355
461
|
const data = hit.route.strategy === 'data';
|
|
356
462
|
const principal = hit.route.scope === 'principal';
|
|
@@ -384,5 +490,18 @@ export function spoolProxy(options) {
|
|
|
384
490
|
});
|
|
385
491
|
return spool.decorate(request, new Response(text, init));
|
|
386
492
|
}
|
|
387
|
-
return Object.assign(handle, {
|
|
493
|
+
return Object.assign(handle, {
|
|
494
|
+
spool,
|
|
495
|
+
trim: stats,
|
|
496
|
+
images: imageStats,
|
|
497
|
+
video: videoStats,
|
|
498
|
+
embeds: embedStats,
|
|
499
|
+
dictionary: dictionaries?.stats ?? {
|
|
500
|
+
registered: 0,
|
|
501
|
+
encoded: 0,
|
|
502
|
+
bytesIn: 0,
|
|
503
|
+
bytesOut: 0,
|
|
504
|
+
skipped: {}
|
|
505
|
+
}
|
|
506
|
+
});
|
|
388
507
|
}
|
package/dist/report/derive.d.ts
CHANGED
|
@@ -23,6 +23,8 @@ export type Fit = {
|
|
|
23
23
|
};
|
|
24
24
|
/** ordinary least squares over at least two points with distinct x */
|
|
25
25
|
export declare function linearFit(points: ReadonlyArray<readonly [number, number]>): Fit;
|
|
26
|
+
/** whether other containers used more than `limit` ms of CPU per second over the window of `row` */
|
|
27
|
+
export declare function isContaminated(row: Row, limit?: number): boolean;
|
|
26
28
|
/** what one operation (a navigation or a request) costs on one arm of a RAPL run */
|
|
27
29
|
export type OpFigures = {
|
|
28
30
|
windows: number;
|
package/dist/report/derive.js
CHANGED
|
@@ -46,6 +46,11 @@ export function linearFit(points) {
|
|
|
46
46
|
}
|
|
47
47
|
const num = (row, key) => typeof row[key] === 'number' ? row[key] : undefined;
|
|
48
48
|
const str = (row, key) => String(row[key] ?? '');
|
|
49
|
+
/** whether other containers used more than `limit` ms of CPU per second over the window of `row` */
|
|
50
|
+
export function isContaminated(row, limit = CONTAMINATION_MS_PER_S) {
|
|
51
|
+
const other = num(row, 'otherCpuMs') ?? num(row, 'otherContainersCpuMs') ?? 0;
|
|
52
|
+
return !(other / (num(row, 'elapsedMs') / 1000) <= limit);
|
|
53
|
+
}
|
|
49
54
|
const sumValues = (value) => typeof value === 'object' && value !== null
|
|
50
55
|
? Object.values(value).reduce((s, v) => s + v, 0)
|
|
51
56
|
: undefined;
|
|
@@ -58,8 +63,7 @@ export function raplRun(rows, options = {}) {
|
|
|
58
63
|
const idleArm = options.idleArm ?? 'idle';
|
|
59
64
|
const limit = options.contamination ?? CONTAMINATION_MS_PER_S;
|
|
60
65
|
const seconds = (row) => num(row, 'elapsedMs') / 1000;
|
|
61
|
-
const
|
|
62
|
-
const kept = rows.filter((row) => other(row) / seconds(row) <= limit);
|
|
66
|
+
const kept = rows.filter((row) => !isContaminated(row, limit));
|
|
63
67
|
const idle = new Map();
|
|
64
68
|
for (const row of kept) {
|
|
65
69
|
if (str(row, 'arm') === idleArm) {
|
package/dist/report/index.d.ts
CHANGED
package/dist/report/index.js
CHANGED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { type Spread } from './derive.js';
|
|
2
|
+
import type { Row } from './measurement.js';
|
|
3
|
+
/** one lever of the ledger: the bytes it saves on each page of a corpus, in page order */
|
|
4
|
+
export type Lever = {
|
|
5
|
+
id: string;
|
|
6
|
+
/** where the lever stands: `default` is on once its parent option is set, `option` needs its own option, `max` is not built or needs a condition the corpus does not show */
|
|
7
|
+
status: 'default' | 'option' | 'max';
|
|
8
|
+
corpus: 'crawl' | 'trim';
|
|
9
|
+
/** the site of each page, in the same order as `bytes` */
|
|
10
|
+
sites: string[];
|
|
11
|
+
bytes: number[];
|
|
12
|
+
};
|
|
13
|
+
/** what a lever saves across a corpus, in bytes */
|
|
14
|
+
export type LeverStats = {
|
|
15
|
+
pages: number;
|
|
16
|
+
withSaving: number;
|
|
17
|
+
medianPage: number;
|
|
18
|
+
/** the median over the pages that save anything, or null when none do */
|
|
19
|
+
medianEligible: number | null;
|
|
20
|
+
/** the median over sites of each site's mean per page */
|
|
21
|
+
medianSite: number;
|
|
22
|
+
mean: number;
|
|
23
|
+
};
|
|
24
|
+
export type CrawlImageRow = {
|
|
25
|
+
kind: 'image';
|
|
26
|
+
site: string;
|
|
27
|
+
profile: string;
|
|
28
|
+
page: string;
|
|
29
|
+
bytes: number;
|
|
30
|
+
format: string;
|
|
31
|
+
arms: {
|
|
32
|
+
native?: {
|
|
33
|
+
bytes: number;
|
|
34
|
+
};
|
|
35
|
+
ceiling?: {
|
|
36
|
+
bytes: number;
|
|
37
|
+
} | null;
|
|
38
|
+
resized?: {
|
|
39
|
+
bytes: number;
|
|
40
|
+
} | null;
|
|
41
|
+
lossless?: {
|
|
42
|
+
bytes: number;
|
|
43
|
+
} | null;
|
|
44
|
+
} | null;
|
|
45
|
+
};
|
|
46
|
+
export type CrawlPageRow = {
|
|
47
|
+
kind: 'page';
|
|
48
|
+
site: string;
|
|
49
|
+
profile: string;
|
|
50
|
+
variant: 'original' | 'lazy';
|
|
51
|
+
page: string;
|
|
52
|
+
ok: true | string;
|
|
53
|
+
imageBytesLoad: number;
|
|
54
|
+
imageBytesScroll: number;
|
|
55
|
+
videoBytes: number;
|
|
56
|
+
videoTags: Array<{
|
|
57
|
+
autoplay: boolean;
|
|
58
|
+
}>;
|
|
59
|
+
iframes: Array<{
|
|
60
|
+
host: string;
|
|
61
|
+
bytes: number;
|
|
62
|
+
}>;
|
|
63
|
+
lcp: {
|
|
64
|
+
tag: string | null;
|
|
65
|
+
loading: string | null;
|
|
66
|
+
} | null;
|
|
67
|
+
};
|
|
68
|
+
export type CrawlEmbedRow = {
|
|
69
|
+
kind: 'embed';
|
|
70
|
+
provider: 'youtube' | 'vimeo';
|
|
71
|
+
arm: 'iframe' | 'facade';
|
|
72
|
+
bytes: number;
|
|
73
|
+
docBytes: number;
|
|
74
|
+
};
|
|
75
|
+
/** the bytes a WebP of each image saves: `native` is the file at its own size, `ceiling` the width ceiling, `resized` the rendered width */
|
|
76
|
+
export declare function imageSavings(rows: readonly CrawlImageRow[]): {
|
|
77
|
+
format: number;
|
|
78
|
+
resize: number;
|
|
79
|
+
lossless: number;
|
|
80
|
+
};
|
|
81
|
+
/** the levers of the image-crawl corpus: the first views of the desktop profile, one entry per page */
|
|
82
|
+
export declare function crawlLevers(rows: readonly Row[]): Lever[];
|
|
83
|
+
/** bytes of image data a visit that never scrolls does not load when `loading="lazy"` is on, by page */
|
|
84
|
+
export declare function lazyUnscrolled(rows: readonly Row[], profile?: string): {
|
|
85
|
+
pairs: number;
|
|
86
|
+
sites: string[];
|
|
87
|
+
bytes: number[];
|
|
88
|
+
baseLoad: number[];
|
|
89
|
+
lazyLoad: number[];
|
|
90
|
+
lcpImagePages: number;
|
|
91
|
+
lcpLazyPages: number;
|
|
92
|
+
};
|
|
93
|
+
/** the levers of the trim corpus: the first views of internal pages of the Drupal sites */
|
|
94
|
+
export declare function trimLevers(trimmed: readonly Row[], crawl: readonly Row[]): Lever[];
|
|
95
|
+
/** {@link LeverStats} of a lever's per-page bytes, or null for an empty corpus */
|
|
96
|
+
export declare function leverStats(sites: readonly string[], bytes: readonly number[]): LeverStats | null;
|
|
97
|
+
/** the bytes of several levers added page by page; the levers must come from one corpus and one page order */
|
|
98
|
+
export declare function addLevers(levers: readonly Lever[]): {
|
|
99
|
+
sites: string[];
|
|
100
|
+
bytes: number[];
|
|
101
|
+
} | null;
|
|
102
|
+
/** the dictionary choices the `dictionary-size` rig measures; see that rig for what each one is */
|
|
103
|
+
export declare const DICTIONARY_ARMS: readonly ["oracle", "browser", "serverVisit", "serverHeld", "serverOut"];
|
|
104
|
+
/** one of {@link DICTIONARY_ARMS} */
|
|
105
|
+
export type DictionaryArm = (typeof DICTIONARY_ARMS)[number];
|
|
106
|
+
/** what one dictionary choice saves against brotli 11 alone, with a file that gains nothing served plain */
|
|
107
|
+
export type DictionaryStat = {
|
|
108
|
+
/** the median over sites of the share of that site's brotli 11 bytes saved */
|
|
109
|
+
medianSite: number;
|
|
110
|
+
/** the same share over every file of every site */
|
|
111
|
+
pooled: number;
|
|
112
|
+
/** internal pages */
|
|
113
|
+
pages: number;
|
|
114
|
+
/** the median bytes saved on an internal page's first view */
|
|
115
|
+
medianPageBytes: number;
|
|
116
|
+
/** internal pages whose new files save more than 5% of their brotli 11 bytes */
|
|
117
|
+
pagesOver5: number;
|
|
118
|
+
};
|
|
119
|
+
/**
|
|
120
|
+
* The savings of each dictionary choice in a `dictionary-size` reading, for `dcz` (zstd) and `dcb`
|
|
121
|
+
* (brotli). A file the choice gives no dictionary, or whose delta is not smaller than brotli 11,
|
|
122
|
+
* saves nothing.
|
|
123
|
+
*/
|
|
124
|
+
export declare function dictionaryArms(rows: readonly Record<string, unknown>[]): {
|
|
125
|
+
sites: number;
|
|
126
|
+
files: number;
|
|
127
|
+
br11Bytes: number;
|
|
128
|
+
arms: Record<"oracle" | "browser" | "serverVisit" | "serverHeld" | "serverOut", {
|
|
129
|
+
zstd: DictionaryStat;
|
|
130
|
+
br: DictionaryStat;
|
|
131
|
+
}>;
|
|
132
|
+
};
|
|
133
|
+
/** the windows of one arm that a derivation keeps and drops, and why */
|
|
134
|
+
export type WindowCount = {
|
|
135
|
+
kept: number;
|
|
136
|
+
/** other containers used more than the limit of CPU per second */
|
|
137
|
+
dropped: number;
|
|
138
|
+
/** the window belongs to a round whose idle windows disagree (idle-activity only) */
|
|
139
|
+
idleControl: number;
|
|
140
|
+
};
|
|
141
|
+
/** kept and dropped windows per arm under the contamination limit; a row with no length (a probe) is not a window */
|
|
142
|
+
export declare function windowCounts(rows: readonly Row[], limit?: number): Record<string, WindowCount>;
|
|
143
|
+
/** the idle windows of a round may differ by this factor, largest over smallest, before the round is dropped */
|
|
144
|
+
export declare const IDLE_AGREEMENT = 1.25;
|
|
145
|
+
/** one arm of an idle-activity reading, after the drops */
|
|
146
|
+
export type ActivityArm = {
|
|
147
|
+
windows: number;
|
|
148
|
+
/** package watts over the kept windows */
|
|
149
|
+
watts: Spread;
|
|
150
|
+
/** watts above the idle windows of the same round */
|
|
151
|
+
extraWatts: Spread;
|
|
152
|
+
};
|
|
153
|
+
/**
|
|
154
|
+
* Package watts per arm of an idle-activity reading. A window is dropped when other containers used
|
|
155
|
+
* more than `limit` ms of CPU per second. The rig cannot see a host process or a container that
|
|
156
|
+
* stopped inside a window, so the three idle windows of each round also act as a control: when the
|
|
157
|
+
* largest of them is over {@link IDLE_AGREEMENT} times the smallest, the whole round is dropped. An
|
|
158
|
+
* arm with no kept window is null.
|
|
159
|
+
*/
|
|
160
|
+
export declare function activityFigures(rows: readonly Row[], limit?: number): {
|
|
161
|
+
idleWatts: Spread | null;
|
|
162
|
+
noisyRounds: number[];
|
|
163
|
+
counts: Record<string, WindowCount>;
|
|
164
|
+
arms: Record<string, ActivityArm | null>;
|
|
165
|
+
};
|
|
166
|
+
/** attributional joules for `bytes` (the same SWD v4 rate as the headline, no origin term: these are the same requests with fewer bytes) */
|
|
167
|
+
export declare const ledgerJoules: (bytes: number) => number;
|
|
168
|
+
/** megawatt-hours a year when `perView` joules are saved on a share of `views` views */
|
|
169
|
+
export declare const ledgerMWh: (perView: number, views: number, share: number) => number;
|