@drupflare/spool 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +263 -41
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/entry.js +33 -1
- package/dist/client/register.d.ts +23 -1
- package/dist/client/register.js +53 -7
- package/dist/protocol/config.d.ts +16 -1
- package/dist/protocol/config.js +3 -1
- package/dist/protocol/dataset.d.ts +183 -0
- package/dist/protocol/dataset.js +206 -0
- package/dist/protocol/envelope.d.ts +1 -0
- package/dist/protocol/envelope.js +2 -1
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/index.d.ts +3 -0
- package/dist/protocol/index.js +3 -0
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +46 -0
- package/dist/protocol/provision.js +38 -0
- package/dist/protocol/query.d.ts +66 -5
- package/dist/protocol/query.js +103 -4
- package/dist/protocol/route.d.ts +6 -2
- package/dist/protocol/route.js +31 -3
- package/dist/protocol/write.d.ts +146 -0
- package/dist/protocol/write.js +101 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +11 -1
- package/dist/report/economics.js +19 -1
- package/dist/report/index.d.ts +2 -0
- package/dist/report/index.js +2 -0
- package/dist/report/inputs.d.ts +97 -0
- package/dist/report/inputs.js +133 -0
- package/dist/report/institution.d.ts +188 -1
- package/dist/report/institution.js +146 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +3 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/dataset.d.ts +48 -0
- package/dist/server/dataset.js +308 -0
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +2 -0
- package/dist/server/spool.d.ts +20 -0
- package/dist/server/spool.js +126 -18
- package/dist/server/write.d.ts +27 -0
- package/dist/server/write.js +127 -0
- package/dist/sw/analytics.d.ts +6 -0
- package/dist/sw/analytics.js +14 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.js +78 -9
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/render.d.ts +7 -2
- package/dist/sw/render.js +19 -5
- package/dist/sw/replica.d.ts +12 -3
- package/dist/sw/replica.js +40 -5
- package/dist/sw/runtime.d.ts +43 -2
- package/dist/sw/runtime.js +340 -24
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +107 -0
- package/dist/sw/write.js +442 -0
- package/package.json +3 -2
|
@@ -1,4 +1,9 @@
|
|
|
1
|
+
import { egridGPerKwh, electricityPrice, tonnesFromMwh, usdForMwh } from './economics.js';
|
|
2
|
+
import { EMBER_US_G_PER_KWH, FIRST_VIEW_BYTES, FIRST_VIEW_SHARE, GOAL_HIGH_SHARE, GOAL_SHARES, HOSTING_KWH_PER_SITE, REPEAT_VIEW_BYTES, SWD_V4, WATER_DIRECT, WATER_INDIRECT, WATER_INDIRECT_LBNL } from './inputs.js';
|
|
1
3
|
import { serverYearMWh } from './model.js';
|
|
4
|
+
import { mismatchesByRoute } from './traffic.js';
|
|
5
|
+
/** what the worker does not count; the trust block reads each as unknown, not as zero */
|
|
6
|
+
const NOT_COUNTED = ['proof failures', 'reseeds', 'rollbacks'];
|
|
2
7
|
/** the priced part of the ledger, or only the unpriced ids when no entry has an energy figure */
|
|
3
8
|
function physicalFrom(ledger) {
|
|
4
9
|
if (ledger.length === 0)
|
|
@@ -48,15 +53,155 @@ function financialFrom(invoices) {
|
|
|
48
53
|
layer: 'L1'
|
|
49
54
|
};
|
|
50
55
|
}
|
|
56
|
+
const SWD_TOTAL = SWD_V4.network.value + SWD_V4.datacentre.value + SWD_V4.device.value;
|
|
57
|
+
/** the share of spool's attributional saving that is data-centre energy: 0.055 of 0.194 kWh/GB */
|
|
58
|
+
const DATA_CENTRE_SHARE = SWD_V4.datacentre.value / SWD_TOTAL;
|
|
59
|
+
const DEVICE_SHARE = SWD_V4.device.value / SWD_TOTAL;
|
|
60
|
+
const NETWORK_SHARE = SWD_V4.network.value / SWD_TOTAL;
|
|
61
|
+
/** joules per byte at the SWD v4 total: kWh per GB, times J per kWh, over bytes per GB */
|
|
62
|
+
const J_PER_BYTE = (SWD_TOTAL * 3.6e6) / 1e9;
|
|
63
|
+
/** joules an average page view costs on the attributional basis: first views at their share, repeat views the rest */
|
|
64
|
+
function footprintJoulesPerView() {
|
|
65
|
+
const first = FIRST_VIEW_SHARE.central.value;
|
|
66
|
+
return J_PER_BYTE * (first * FIRST_VIEW_BYTES.value + (1 - first) * REPEAT_VIEW_BYTES.value);
|
|
67
|
+
}
|
|
68
|
+
/** the page-view footprint of `viewsPerYear` views, in MWh a year, on the attributional basis */
|
|
69
|
+
export function footprintMwh(viewsPerYear) {
|
|
70
|
+
return (viewsPerYear * footprintJoulesPerView()) / 3.6e9;
|
|
71
|
+
}
|
|
72
|
+
function economicsFrom(profile) {
|
|
73
|
+
if (!profile)
|
|
74
|
+
return null;
|
|
75
|
+
const industrial = electricityPrice(profile.state, 'industrial');
|
|
76
|
+
const commercial = electricityPrice(profile.state, 'commercial');
|
|
77
|
+
const dataCentreMwh = profile.attributionalMwh * DATA_CENTRE_SHARE;
|
|
78
|
+
return {
|
|
79
|
+
state: profile.state,
|
|
80
|
+
industrial,
|
|
81
|
+
commercial,
|
|
82
|
+
cases: [
|
|
83
|
+
['peak', HOSTING_KWH_PER_SITE.peak],
|
|
84
|
+
['production', HOSTING_KWH_PER_SITE.production],
|
|
85
|
+
['floor', HOSTING_KWH_PER_SITE.floor]
|
|
86
|
+
].map(([name, perSite]) => {
|
|
87
|
+
const mwh = (profile.sites * perSite.value) / 1000;
|
|
88
|
+
return {
|
|
89
|
+
name,
|
|
90
|
+
kwhPerSite: perSite.value,
|
|
91
|
+
mwh,
|
|
92
|
+
usdIndustrial: usdForMwh(mwh, industrial.value),
|
|
93
|
+
usdCommercial: usdForMwh(mwh, commercial.value)
|
|
94
|
+
};
|
|
95
|
+
}),
|
|
96
|
+
dataCentreMwh,
|
|
97
|
+
offMeterMwh: profile.attributionalMwh - dataCentreMwh,
|
|
98
|
+
layer: 'L2'
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
function storageFrom(inputs) {
|
|
102
|
+
if (!inputs)
|
|
103
|
+
return null;
|
|
104
|
+
for (const h of inputs.holdings) {
|
|
105
|
+
if (!Number.isFinite(h.bytes) || h.bytes < 0) {
|
|
106
|
+
throw new RangeError(`${h.dataset}: bytes must be a non-negative finite number`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
const held = { device: 0, server: 0 };
|
|
110
|
+
for (const h of inputs.holdings)
|
|
111
|
+
held[h.where] += h.bytes;
|
|
112
|
+
return {
|
|
113
|
+
budget: inputs.budget,
|
|
114
|
+
held,
|
|
115
|
+
within: {
|
|
116
|
+
device: held.device <= inputs.budget.device,
|
|
117
|
+
server: held.server <= inputs.budget.server
|
|
118
|
+
},
|
|
119
|
+
holdings: inputs.holdings,
|
|
120
|
+
layer: 'L1'
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
function trustFrom(views) {
|
|
124
|
+
if (!views)
|
|
125
|
+
return null;
|
|
126
|
+
const routes = mismatchesByRoute(views);
|
|
127
|
+
return {
|
|
128
|
+
views: views.length,
|
|
129
|
+
sampled: routes.reduce((n, r) => n + r.compared, 0),
|
|
130
|
+
mismatches: routes.reduce((n, r) => n + r.mismatches, 0),
|
|
131
|
+
refused: views.filter((v) => v.reason === 'refused' || v.reason === 'hard-refused').length,
|
|
132
|
+
fallbacks: views.filter((v) => v.reason === 'invalidated' || v.reason === 'stale-after-sync').length,
|
|
133
|
+
notCounted: NOT_COUNTED,
|
|
134
|
+
layer: 'L1'
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
function waterLitres(dataCentreMwh) {
|
|
138
|
+
const kwh = dataCentreMwh * 1000;
|
|
139
|
+
return {
|
|
140
|
+
directLitres: kwh * WATER_DIRECT.value,
|
|
141
|
+
indirectLitres: kwh * WATER_INDIRECT.value,
|
|
142
|
+
indirectLbnlLitres: kwh * WATER_INDIRECT_LBNL.value
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
function researchFrom(profile) {
|
|
146
|
+
if (!profile)
|
|
147
|
+
return null;
|
|
148
|
+
const att = profile.attributionalMwh;
|
|
149
|
+
const gPerKwh = egridGPerKwh(profile.subregion);
|
|
150
|
+
const footprint = footprintMwh(profile.viewsPerYear);
|
|
151
|
+
const goal = (share) => ({ share, kwh: footprint * share * 1000 });
|
|
152
|
+
return {
|
|
153
|
+
boundary: {
|
|
154
|
+
devices: { mwh: att * DEVICE_SHARE, share: DEVICE_SHARE },
|
|
155
|
+
networks: { mwh: att * NETWORK_SHARE, share: NETWORK_SHARE },
|
|
156
|
+
dataCentres: { mwh: att * DATA_CENTRE_SHARE, share: DATA_CENTRE_SHARE }
|
|
157
|
+
},
|
|
158
|
+
carbon: {
|
|
159
|
+
subregion: profile.subregion,
|
|
160
|
+
gPerKwh,
|
|
161
|
+
tonnes: tonnesFromMwh(att, gPerKwh),
|
|
162
|
+
ember: {
|
|
163
|
+
gPerKwh: EMBER_US_G_PER_KWH.value,
|
|
164
|
+
tonnes: tonnesFromMwh(att, EMBER_US_G_PER_KWH.value)
|
|
165
|
+
}
|
|
166
|
+
},
|
|
167
|
+
water: waterLitres(att * DATA_CENTRE_SHARE),
|
|
168
|
+
goal: {
|
|
169
|
+
viewsPerYear: profile.viewsPerYear,
|
|
170
|
+
footprintMwh: footprint,
|
|
171
|
+
rungs: GOAL_SHARES.map(goal),
|
|
172
|
+
high: goal(GOAL_HIGH_SHARE.value)
|
|
173
|
+
},
|
|
174
|
+
layer: 'L2'
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
/** sums energy and carbon across institutions and keeps water per institution */
|
|
178
|
+
export function rollUpInstitutions(rows) {
|
|
179
|
+
let mwh = 0;
|
|
180
|
+
let tonnes = 0;
|
|
181
|
+
const water = rows.map((row) => {
|
|
182
|
+
mwh += row.attributionalMwh;
|
|
183
|
+
tonnes += tonnesFromMwh(row.attributionalMwh, egridGPerKwh(row.subregion));
|
|
184
|
+
const { directLitres, indirectLitres } = waterLitres(row.attributionalMwh * DATA_CENTRE_SHARE);
|
|
185
|
+
return { name: row.name, directLitres, indirectLitres };
|
|
186
|
+
});
|
|
187
|
+
return { institutions: rows.length, mwh, tonnes, water };
|
|
188
|
+
}
|
|
51
189
|
export function institutionReport(inputs) {
|
|
52
190
|
const ledger = inputs.ledger ?? [];
|
|
53
191
|
const potential = inputs.plan
|
|
54
192
|
? { plan: inputs.plan, layer: inputs.planLayer ?? 'L3' }
|
|
55
193
|
: null;
|
|
194
|
+
const blocks = {
|
|
195
|
+
economics: economicsFrom(inputs.institution),
|
|
196
|
+
storage: storageFrom(inputs.storage),
|
|
197
|
+
trust: trustFrom(inputs.views),
|
|
198
|
+
research: researchFrom(inputs.institution)
|
|
199
|
+
};
|
|
56
200
|
const found = physicalFrom(ledger);
|
|
57
201
|
if (!found || !('mwh' in found)) {
|
|
58
202
|
const unpriced = found?.unpriced ?? [];
|
|
59
203
|
return {
|
|
204
|
+
...blocks,
|
|
60
205
|
observed: observedFrom(inputs.observed),
|
|
61
206
|
potential,
|
|
62
207
|
financial: financialFrom(inputs.invoices),
|
|
@@ -70,6 +215,7 @@ export function institutionReport(inputs) {
|
|
|
70
215
|
const years = servers === '1' ? 'server-year' : 'server-years';
|
|
71
216
|
const entries = priced === 1 ? 'entry' : 'entries';
|
|
72
217
|
return {
|
|
218
|
+
...blocks,
|
|
73
219
|
observed: observedFrom(inputs.observed),
|
|
74
220
|
potential,
|
|
75
221
|
financial: financialFrom(inputs.invoices),
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { Row } from './measurement.js';
|
|
2
|
+
/** seconds in each window of the avoidable-loops crawl */
|
|
3
|
+
export declare const LOOP_WINDOW_S = 20;
|
|
4
|
+
/** the classes a page view can carry, in the order the table prints them */
|
|
5
|
+
export declare const LOOP_CLASSES: readonly ["visible-frame-loop", "hidden-frame-loop", "hidden-timer-loop", "poll", "autoplay", "css-animation", "any", "hidden-timer-any"];
|
|
6
|
+
export type LoopClass = (typeof LOOP_CLASSES)[number];
|
|
7
|
+
/** one second of a window: what the page did in it, as the crawl counted it */
|
|
8
|
+
export type LoopSecond = {
|
|
9
|
+
visibility: string;
|
|
10
|
+
raf: number;
|
|
11
|
+
timers: {
|
|
12
|
+
lt100: number;
|
|
13
|
+
lt1s: number;
|
|
14
|
+
gte1s: number;
|
|
15
|
+
};
|
|
16
|
+
calls: {
|
|
17
|
+
fetch: number;
|
|
18
|
+
xhr: number;
|
|
19
|
+
beacon: number;
|
|
20
|
+
websocket: number;
|
|
21
|
+
eventSource: number;
|
|
22
|
+
};
|
|
23
|
+
requests: {
|
|
24
|
+
same: Record<string, number>;
|
|
25
|
+
cross: Record<string, number>;
|
|
26
|
+
};
|
|
27
|
+
videoAdvancing: number;
|
|
28
|
+
infiniteAnimations: number;
|
|
29
|
+
/** the tab's state at the read: whether Chromium discarded its document, its readyState, and whether its URL changed since the window's first read */
|
|
30
|
+
state?: {
|
|
31
|
+
discarded: boolean;
|
|
32
|
+
ready: string;
|
|
33
|
+
moved: boolean;
|
|
34
|
+
};
|
|
35
|
+
};
|
|
36
|
+
export type LoopWindow = {
|
|
37
|
+
seconds: LoopSecond[];
|
|
38
|
+
};
|
|
39
|
+
/** the classes of one page, null for a page that did not load; a null class could not be measured */
|
|
40
|
+
export declare function loopClasses(page: Row): Record<LoopClass, boolean | null> | null;
|
|
41
|
+
/** Wilson score interval for `k` of `n` at 95%, null when `n` is 0 */
|
|
42
|
+
export declare function wilson(k: number, n: number): {
|
|
43
|
+
low: number;
|
|
44
|
+
high: number;
|
|
45
|
+
} | null;
|
|
46
|
+
export type LoopFigure = {
|
|
47
|
+
/** pages where the class could be measured */
|
|
48
|
+
n: number;
|
|
49
|
+
count: number;
|
|
50
|
+
share: number | null;
|
|
51
|
+
low: number | null;
|
|
52
|
+
high: number | null;
|
|
53
|
+
/** sites with a measured page for the class, and sites with at least one page that has it */
|
|
54
|
+
siteN: number;
|
|
55
|
+
siteCount: number;
|
|
56
|
+
};
|
|
57
|
+
export type LoopFigures = {
|
|
58
|
+
/** pages that loaded */
|
|
59
|
+
pages: number;
|
|
60
|
+
sites: number;
|
|
61
|
+
classes: Record<LoopClass, LoopFigure>;
|
|
62
|
+
};
|
|
63
|
+
/** the share of loaded pages, and of sites, that carry each class; `slice` takes the page rows by position */
|
|
64
|
+
export declare function loopFigures(rows: readonly Row[], slice?: {
|
|
65
|
+
from: number;
|
|
66
|
+
to: number;
|
|
67
|
+
}): LoopFigures;
|
|
68
|
+
/** the figures of all the page rows, and of the first `floor(n / 2)` and the rest, in row order */
|
|
69
|
+
export declare function halves(rows: readonly Row[]): {
|
|
70
|
+
all: LoopFigures;
|
|
71
|
+
first: LoopFigures;
|
|
72
|
+
second: LoopFigures;
|
|
73
|
+
};
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/** seconds in each window of the avoidable-loops crawl */
|
|
2
|
+
export const LOOP_WINDOW_S = 20;
|
|
3
|
+
/** the window counts only when its page was in the state it was recorded in for this many seconds */
|
|
4
|
+
const VALID_SECONDS = 18;
|
|
5
|
+
/** a class holds when its condition holds on this many seconds of the window */
|
|
6
|
+
const NEEDED_SECONDS = 15;
|
|
7
|
+
const VISIBLE_FRAMES_PER_S = 20;
|
|
8
|
+
const HIDDEN_FRAMES_PER_S = 1;
|
|
9
|
+
const POLL_REQUESTS = 3;
|
|
10
|
+
/** the classes of the six primary loops, which `any` is the union of */
|
|
11
|
+
const PRIMARY = [
|
|
12
|
+
'visible-frame-loop',
|
|
13
|
+
'hidden-frame-loop',
|
|
14
|
+
'hidden-timer-loop',
|
|
15
|
+
'poll',
|
|
16
|
+
'autoplay',
|
|
17
|
+
'css-animation'
|
|
18
|
+
];
|
|
19
|
+
/** the classes a page view can carry, in the order the table prints them */
|
|
20
|
+
export const LOOP_CLASSES = [...PRIMARY, 'any', 'hidden-timer-any'];
|
|
21
|
+
/** a window that holds `state` for enough of its seconds to count; null otherwise */
|
|
22
|
+
function usable(win, state) {
|
|
23
|
+
if (!win)
|
|
24
|
+
return null;
|
|
25
|
+
return win.seconds.filter((s) => s.visibility === state).length >= VALID_SECONDS ? win : null;
|
|
26
|
+
}
|
|
27
|
+
/** whether `test` holds for enough seconds of a usable window, null when the window is not usable */
|
|
28
|
+
function over(win, test, needed) {
|
|
29
|
+
if (!win)
|
|
30
|
+
return null;
|
|
31
|
+
return win.seconds.filter(test).length >= needed;
|
|
32
|
+
}
|
|
33
|
+
/** the classes of one page, null for a page that did not load; a null class could not be measured */
|
|
34
|
+
export function loopClasses(page) {
|
|
35
|
+
if (page.ok !== true)
|
|
36
|
+
return null;
|
|
37
|
+
const visible = usable(page.visible, 'visible');
|
|
38
|
+
const hidden = usable(page.hidden, 'hidden');
|
|
39
|
+
const requests = (s) => Object.values(s.requests.same).reduce((n, c) => n + c, 0) +
|
|
40
|
+
Object.values(s.requests.cross).reduce((n, c) => n + c, 0);
|
|
41
|
+
const classes = {
|
|
42
|
+
'visible-frame-loop': over(visible, (s) => s.raf >= VISIBLE_FRAMES_PER_S, NEEDED_SECONDS),
|
|
43
|
+
'hidden-frame-loop': over(hidden, (s) => s.raf >= HIDDEN_FRAMES_PER_S, NEEDED_SECONDS),
|
|
44
|
+
'hidden-timer-loop': over(hidden, (s) => s.timers.gte1s >= 1, NEEDED_SECONDS),
|
|
45
|
+
poll: visible === null
|
|
46
|
+
? null
|
|
47
|
+
: visible.seconds.reduce((n, s) => n + requests(s), 0) >= POLL_REQUESTS,
|
|
48
|
+
autoplay: over(visible, (s) => s.videoAdvancing > 0, 1),
|
|
49
|
+
'css-animation': over(visible, (s) => s.infiniteAnimations > 0, 1),
|
|
50
|
+
'hidden-timer-any': over(hidden, (s) => s.timers.lt100 + s.timers.lt1s + s.timers.gte1s >= 1, NEEDED_SECONDS),
|
|
51
|
+
any: null
|
|
52
|
+
};
|
|
53
|
+
const values = PRIMARY.map((c) => classes[c]);
|
|
54
|
+
classes.any = values.includes(true) ? true : values.includes(null) ? null : false;
|
|
55
|
+
return classes;
|
|
56
|
+
}
|
|
57
|
+
/** Wilson score interval for `k` of `n` at 95%, null when `n` is 0 */
|
|
58
|
+
export function wilson(k, n) {
|
|
59
|
+
if (n <= 0)
|
|
60
|
+
return null;
|
|
61
|
+
const z = 1.959964;
|
|
62
|
+
const z2 = z * z;
|
|
63
|
+
const p = k / n;
|
|
64
|
+
const center = (p + z2 / (2 * n)) / (1 + z2 / n);
|
|
65
|
+
const half = (z * Math.sqrt((p * (1 - p)) / n + z2 / (4 * n * n))) / (1 + z2 / n);
|
|
66
|
+
return { low: Math.max(0, center - half), high: Math.min(1, center + half) };
|
|
67
|
+
}
|
|
68
|
+
/** the share of loaded pages, and of sites, that carry each class; `slice` takes the page rows by position */
|
|
69
|
+
export function loopFigures(rows, slice) {
|
|
70
|
+
const pages = rows.filter((row) => row.kind === 'page').slice(slice?.from, slice?.to);
|
|
71
|
+
const loaded = pages.flatMap((row) => {
|
|
72
|
+
const classes = loopClasses(row);
|
|
73
|
+
return classes === null ? [] : [{ site: String(row.site), classes }];
|
|
74
|
+
});
|
|
75
|
+
const out = {};
|
|
76
|
+
for (const cls of LOOP_CLASSES) {
|
|
77
|
+
const measured = loaded.filter((p) => p.classes[cls] !== null);
|
|
78
|
+
const hits = measured.filter((p) => p.classes[cls] === true);
|
|
79
|
+
const interval = wilson(hits.length, measured.length);
|
|
80
|
+
out[cls] = {
|
|
81
|
+
n: measured.length,
|
|
82
|
+
count: hits.length,
|
|
83
|
+
share: measured.length > 0 ? hits.length / measured.length : null,
|
|
84
|
+
low: interval?.low ?? null,
|
|
85
|
+
high: interval?.high ?? null,
|
|
86
|
+
siteN: new Set(measured.map((p) => p.site)).size,
|
|
87
|
+
siteCount: new Set(hits.map((p) => p.site)).size
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
return { pages: loaded.length, sites: new Set(loaded.map((p) => p.site)).size, classes: out };
|
|
91
|
+
}
|
|
92
|
+
/** the figures of all the page rows, and of the first `floor(n / 2)` and the rest, in row order */
|
|
93
|
+
export function halves(rows) {
|
|
94
|
+
const n = rows.filter((row) => row.kind === 'page').length;
|
|
95
|
+
const cut = Math.floor(n / 2);
|
|
96
|
+
return {
|
|
97
|
+
all: loopFigures(rows),
|
|
98
|
+
first: loopFigures(rows, { from: 0, to: cut }),
|
|
99
|
+
second: loopFigures(rows, { from: cut, to: n })
|
|
100
|
+
};
|
|
101
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Layer } from './inputs.js';
|
|
2
2
|
/** the rigs that write to `measurements/<rig>/`, one folder each */
|
|
3
|
-
export declare const RIGS: readonly ["repeat-nav", "on-wire", "zero-network", "device-cpu", "rapl", "iphone", "payload", "trim", "link-assets", "image-crawl", "dictionary-size", "nav-replay", "first-payload", "embed-markup"];
|
|
3
|
+
export declare const RIGS: readonly ["repeat-nav", "on-wire", "zero-network", "device-cpu", "rapl", "iphone", "payload", "trim", "link-assets", "image-crawl", "dictionary-size", "nav-replay", "first-payload", "embed-markup", "dataset-baseline", "avoidable-loops"];
|
|
4
4
|
/** one of {@link RIGS} */
|
|
5
5
|
export type Rig = (typeof RIGS)[number];
|
|
6
6
|
/** the format version {@link parseMeasurement} reads */
|
|
@@ -13,7 +13,9 @@ export const RIGS = [
|
|
|
13
13
|
'dictionary-size',
|
|
14
14
|
'nav-replay',
|
|
15
15
|
'first-payload',
|
|
16
|
-
'embed-markup'
|
|
16
|
+
'embed-markup',
|
|
17
|
+
'dataset-baseline',
|
|
18
|
+
'avoidable-loops'
|
|
17
19
|
];
|
|
18
20
|
/** the format version {@link parseMeasurement} reads */
|
|
19
21
|
export const MEASUREMENT_SCHEMA = 1;
|
package/dist/report/model.d.ts
CHANGED
|
@@ -23,6 +23,8 @@ export declare function joulesPerViewWithOrigin(originJ: number, terms?: Pick<Vi
|
|
|
23
23
|
export declare function dataCentreJoules(bytes?: number): number;
|
|
24
24
|
/** the views channel: annual views x local share x joules per local view, in MWh */
|
|
25
25
|
export declare function viewsChannelMWh(annualViews: number, localShare: number, jPerView?: number): number;
|
|
26
|
+
/** the reporting unit: kWh per million views when each view takes `joules` */
|
|
27
|
+
export declare function kwhPerMv(joules: number): number;
|
|
26
28
|
/** a year of one server's facility draw, in MWh */
|
|
27
29
|
export declare function serverYearMWh(watts?: number, pue?: number): number;
|
|
28
30
|
/** origin work removed at peak: local share x the share of load that is reads */
|
package/dist/report/model.js
CHANGED
|
@@ -38,6 +38,10 @@ export function dataCentreJoules(bytes = DOCUMENT_BYTES.value) {
|
|
|
38
38
|
export function viewsChannelMWh(annualViews, localShare, jPerView = joulesPerView()) {
|
|
39
39
|
return (annualViews * localShare * jPerView) / J_PER_MWH;
|
|
40
40
|
}
|
|
41
|
+
/** the reporting unit: kWh per million views when each view takes `joules` */
|
|
42
|
+
export function kwhPerMv(joules) {
|
|
43
|
+
return (joules * 1e6) / J_PER_KWH;
|
|
44
|
+
}
|
|
41
45
|
/** a year of one server's facility draw, in MWh */
|
|
42
46
|
export function serverYearMWh(watts = SERVER_WATTS.value, pue = PUE.value) {
|
|
43
47
|
return (watts * pue * HOURS_PER_YEAR) / 1e6;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { TimedView } from '../protocol/view.js';
|
|
2
|
+
import type { Row } from './measurement.js';
|
|
2
3
|
/** what a third-party origin is for, as far as the table below knows */
|
|
3
4
|
export type OriginClass = 'analytics' | 'ads' | 'consent' | 'fonts' | 'cdn' | 'video' | 'maps' | 'social' | 'chat' | 'unknown';
|
|
4
5
|
/**
|
|
@@ -53,6 +54,62 @@ export type ThirdPartyReport = {
|
|
|
53
54
|
/** origins by `soloViews`, then `views`, then name */
|
|
54
55
|
rows: ThirdPartyRow[];
|
|
55
56
|
};
|
|
57
|
+
/** how long a third-party response may be kept without asking its origin */
|
|
58
|
+
export type Cacheability = 'immutable' | 'long' | 'short' | 'uncacheable';
|
|
59
|
+
/**
|
|
60
|
+
* The class of one recorded response from its cache headers. `immutable` is an `immutable`
|
|
61
|
+
* directive or a max-age of a year; `long` a max-age of a day up to a year; `short` a max-age under
|
|
62
|
+
* a day, or a validator with no freshness; `uncacheable` a `no-store`, or neither freshness nor a
|
|
63
|
+
* validator. The reading keeps `expires` as a flag only, so it is not read.
|
|
64
|
+
*/
|
|
65
|
+
export declare function cacheability(response: Row): Cacheability;
|
|
66
|
+
/** third-party network bytes and requests of one repeat view, by cacheability */
|
|
67
|
+
export type CacheSplit = {
|
|
68
|
+
bytes: Record<Cacheability, number>;
|
|
69
|
+
requests: Record<Cacheability, number>;
|
|
70
|
+
/** third-party network bytes */
|
|
71
|
+
total: number;
|
|
72
|
+
/** bytes from a cache-remedy origin that is immutable or long: what a worker could answer */
|
|
73
|
+
answerable: number;
|
|
74
|
+
/** bytes, and immutable or long bytes, by origin class and remedy, keyed `class|remedy` */
|
|
75
|
+
byOrigin: Record<string, {
|
|
76
|
+
bytes: number;
|
|
77
|
+
longLived: number;
|
|
78
|
+
}>;
|
|
79
|
+
};
|
|
80
|
+
/**
|
|
81
|
+
* The third-party requests of one repeat view that reached the network, split by cacheability.
|
|
82
|
+
* First-party requests and requests the browser cache answered are left out, as in `splitRequestBytes`.
|
|
83
|
+
*/
|
|
84
|
+
export declare function thirdPartyCacheSplit(requests: readonly Row[]): CacheSplit;
|
|
85
|
+
/** {@link thirdPartyCacheSplit} for each site's loaded home-page repeat view, picked as `repeatSplitBySite` picks it */
|
|
86
|
+
export declare function thirdPartyCacheBySite(rows: readonly Row[]): Map<string, CacheSplit>;
|
|
87
|
+
/** cross-site figures of {@link CacheSplit}s; shares are over the sites that had third-party bytes */
|
|
88
|
+
export type CacheabilityFigures = {
|
|
89
|
+
/** sites with a loaded repeat view */
|
|
90
|
+
sites: number;
|
|
91
|
+
/** sites whose repeat view had third-party network bytes */
|
|
92
|
+
withThirdParty: number;
|
|
93
|
+
/** each class's share of a site's third-party bytes, and its pooled share */
|
|
94
|
+
shares: Record<Cacheability, {
|
|
95
|
+
median: number;
|
|
96
|
+
mean: number;
|
|
97
|
+
pooled: number;
|
|
98
|
+
}>;
|
|
99
|
+
/** answerable bytes per site, over every site, zero included */
|
|
100
|
+
answerableBytes: {
|
|
101
|
+
median: number;
|
|
102
|
+
mean: number;
|
|
103
|
+
};
|
|
104
|
+
/** answerable bytes as a share of a site's third-party bytes, and pooled */
|
|
105
|
+
answerableShare: {
|
|
106
|
+
median: number;
|
|
107
|
+
mean: number;
|
|
108
|
+
pooled: number;
|
|
109
|
+
};
|
|
110
|
+
};
|
|
111
|
+
/** the median, mean and pooled figures of the splits, one per class and for the answerable bytes */
|
|
112
|
+
export declare function cacheabilityFigures(splits: readonly CacheSplit[]): CacheabilityFigures;
|
|
56
113
|
/**
|
|
57
114
|
* What blocks zero-network on this site: for each third-party origin the views that loaded from it,
|
|
58
115
|
* ranked by how many views it alone blocks. Pass the views from `viewsOf`, which joins each page's
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { loaded, median } from './derive.js';
|
|
1
2
|
/**
|
|
2
3
|
* A short table of common third-party hosts. It is a starting point for a site to extend, not a
|
|
3
4
|
* claim about any vendor's behaviour: the class and remedy come from reading vendor documentation
|
|
@@ -48,6 +49,116 @@ export function classifyOrigin(origin, table = KNOWN_ORIGINS) {
|
|
|
48
49
|
? { class: best.class, remedy: best.remedy }
|
|
49
50
|
: { class: 'unknown', remedy: 'review' };
|
|
50
51
|
}
|
|
52
|
+
const DAY_S = 86400;
|
|
53
|
+
const YEAR_S = 365 * DAY_S;
|
|
54
|
+
const CLASSES = ['immutable', 'long', 'short', 'uncacheable'];
|
|
55
|
+
/**
|
|
56
|
+
* The class of one recorded response from its cache headers. `immutable` is an `immutable`
|
|
57
|
+
* directive or a max-age of a year; `long` a max-age of a day up to a year; `short` a max-age under
|
|
58
|
+
* a day, or a validator with no freshness; `uncacheable` a `no-store`, or neither freshness nor a
|
|
59
|
+
* validator. The reading keeps `expires` as a flag only, so it is not read.
|
|
60
|
+
*/
|
|
61
|
+
export function cacheability(response) {
|
|
62
|
+
const directives = new Map();
|
|
63
|
+
for (const part of String(response.cacheControl ?? '').split(',')) {
|
|
64
|
+
const [name = '', value = ''] = part.trim().toLowerCase().split('=');
|
|
65
|
+
if (name)
|
|
66
|
+
directives.set(name, value.replace(/"/g, ''));
|
|
67
|
+
}
|
|
68
|
+
if (directives.has('no-store'))
|
|
69
|
+
return 'uncacheable';
|
|
70
|
+
const raw = directives.get('max-age');
|
|
71
|
+
const maxAge = raw === undefined ? NaN : Number(raw);
|
|
72
|
+
if (directives.has('immutable') || maxAge >= YEAR_S)
|
|
73
|
+
return 'immutable';
|
|
74
|
+
if (maxAge >= DAY_S)
|
|
75
|
+
return 'long';
|
|
76
|
+
if (Number.isFinite(maxAge))
|
|
77
|
+
return 'short';
|
|
78
|
+
return response.etag === true || response.lastModified === true ? 'short' : 'uncacheable';
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* The third-party requests of one repeat view that reached the network, split by cacheability.
|
|
82
|
+
* First-party requests and requests the browser cache answered are left out, as in `splitRequestBytes`.
|
|
83
|
+
*/
|
|
84
|
+
export function thirdPartyCacheSplit(requests) {
|
|
85
|
+
const zeros = () => ({
|
|
86
|
+
immutable: 0,
|
|
87
|
+
long: 0,
|
|
88
|
+
short: 0,
|
|
89
|
+
uncacheable: 0
|
|
90
|
+
});
|
|
91
|
+
const split = {
|
|
92
|
+
bytes: zeros(),
|
|
93
|
+
requests: zeros(),
|
|
94
|
+
total: 0,
|
|
95
|
+
answerable: 0,
|
|
96
|
+
byOrigin: {}
|
|
97
|
+
};
|
|
98
|
+
for (const request of requests) {
|
|
99
|
+
if (request.fromCache === true || request.sameOrigin === true)
|
|
100
|
+
continue;
|
|
101
|
+
const bytes = typeof request.bytes === 'number' ? request.bytes : 0;
|
|
102
|
+
const kind = cacheability(request);
|
|
103
|
+
const { class: origin, remedy } = classifyOrigin(String(request.url ?? ''));
|
|
104
|
+
const longLived = kind === 'immutable' || kind === 'long' ? bytes : 0;
|
|
105
|
+
split.bytes[kind] += bytes;
|
|
106
|
+
split.requests[kind] += 1;
|
|
107
|
+
split.total += bytes;
|
|
108
|
+
if (remedy === 'cache')
|
|
109
|
+
split.answerable += longLived;
|
|
110
|
+
const key = `${origin}|${remedy}`;
|
|
111
|
+
const cell = split.byOrigin[key] ?? { bytes: 0, longLived: 0 };
|
|
112
|
+
cell.bytes += bytes;
|
|
113
|
+
cell.longLived += longLived;
|
|
114
|
+
split.byOrigin[key] = cell;
|
|
115
|
+
}
|
|
116
|
+
return split;
|
|
117
|
+
}
|
|
118
|
+
/** {@link thirdPartyCacheSplit} for each site's loaded home-page repeat view, picked as `repeatSplitBySite` picks it */
|
|
119
|
+
export function thirdPartyCacheBySite(rows) {
|
|
120
|
+
const out = new Map();
|
|
121
|
+
for (const row of rows) {
|
|
122
|
+
for (const nav of row.navs ?? []) {
|
|
123
|
+
if (loaded(nav) && nav.kind === 'warm-home-repeat' && Array.isArray(nav.requestList)) {
|
|
124
|
+
out.set(String(row.site ?? ''), thirdPartyCacheSplit(nav.requestList));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return out;
|
|
129
|
+
}
|
|
130
|
+
const ratio = (part, whole) => (whole === 0 ? 0 : part / whole);
|
|
131
|
+
const mean = (xs) => xs.length === 0 ? 0 : xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
132
|
+
const middle = (xs) => (xs.length === 0 ? 0 : median(xs));
|
|
133
|
+
/** the median, mean and pooled figures of the splits, one per class and for the answerable bytes */
|
|
134
|
+
export function cacheabilityFigures(splits) {
|
|
135
|
+
const timed = splits.filter((s) => s.total > 0);
|
|
136
|
+
const pooledTotal = splits.reduce((n, s) => n + s.total, 0);
|
|
137
|
+
const shares = {};
|
|
138
|
+
for (const kind of CLASSES) {
|
|
139
|
+
const xs = timed.map((s) => s.bytes[kind] / s.total);
|
|
140
|
+
shares[kind] = {
|
|
141
|
+
median: middle(xs),
|
|
142
|
+
mean: mean(xs),
|
|
143
|
+
pooled: ratio(splits.reduce((n, s) => n + s.bytes[kind], 0), pooledTotal)
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
const answerable = timed.map((s) => s.answerable / s.total);
|
|
147
|
+
return {
|
|
148
|
+
sites: splits.length,
|
|
149
|
+
withThirdParty: timed.length,
|
|
150
|
+
shares,
|
|
151
|
+
answerableBytes: {
|
|
152
|
+
median: middle(splits.map((s) => s.answerable)),
|
|
153
|
+
mean: mean(splits.map((s) => s.answerable))
|
|
154
|
+
},
|
|
155
|
+
answerableShare: {
|
|
156
|
+
median: middle(answerable),
|
|
157
|
+
mean: mean(answerable),
|
|
158
|
+
pooled: ratio(splits.reduce((n, s) => n + s.answerable, 0), pooledTotal)
|
|
159
|
+
}
|
|
160
|
+
};
|
|
161
|
+
}
|
|
51
162
|
/**
|
|
52
163
|
* What blocks zero-network on this site: for each third-party origin the views that loaded from it,
|
|
53
164
|
* ranked by how many views it alone blocks. Pass the views from `viewsOf`, which joins each page's
|
package/dist/report/trim.d.ts
CHANGED
|
@@ -75,6 +75,62 @@ export type CoverageFigures = {
|
|
|
75
75
|
jsLoad: Quartiles | null;
|
|
76
76
|
jsScroll: Quartiles | null;
|
|
77
77
|
};
|
|
78
|
+
/** a changed aggregate's comparison with an earlier file of its kind; the zstd sizes are null when the base did not load */
|
|
79
|
+
export type Base = {
|
|
80
|
+
url: string | null;
|
|
81
|
+
status: number | null;
|
|
82
|
+
zstdAlone: number | null;
|
|
83
|
+
zstdDict: number | null;
|
|
84
|
+
};
|
|
85
|
+
/** one same-origin CSS or JS aggregate of a site, keyed by its path with the query dropped */
|
|
86
|
+
export type Aggregate = {
|
|
87
|
+
path: string;
|
|
88
|
+
kind: 'css' | 'js';
|
|
89
|
+
raw: number;
|
|
90
|
+
/** the largest transfer a page recorded for it, or null when no page did */
|
|
91
|
+
transfer: number | null;
|
|
92
|
+
/** only on a reading taken with an earlier reading to compare against, for a name that changed */
|
|
93
|
+
base: Base | null;
|
|
94
|
+
};
|
|
95
|
+
/** whether a URL is a Drupal aggregate: a CSS or JS file directly under a `files/css/` or `files/js/` folder */
|
|
96
|
+
export declare function isAggregate(url: string): boolean;
|
|
97
|
+
/** the aggregates of each site of a trim reading, one entry per path across its pages */
|
|
98
|
+
export declare function aggregatesOf(rows: readonly Row[]): Map<string, Aggregate[]>;
|
|
99
|
+
/** one site's aggregates on the second reading against the first */
|
|
100
|
+
export type ChurnSite = {
|
|
101
|
+
site: string;
|
|
102
|
+
/** aggregates on the second reading */
|
|
103
|
+
aggregates: number;
|
|
104
|
+
/** of which the name is new since the first reading */
|
|
105
|
+
changed: number;
|
|
106
|
+
/** the changed names' share of the second reading's aggregate transfer bytes; null without any transfer */
|
|
107
|
+
changedShare: number | null;
|
|
108
|
+
/** changed aggregates whose base loaded, and the zstd dictionary size over standalone size across them */
|
|
109
|
+
compared: number;
|
|
110
|
+
dictRatio: number | null;
|
|
111
|
+
/** the same non-empty set of names on both readings */
|
|
112
|
+
unchanged: boolean;
|
|
113
|
+
};
|
|
114
|
+
/** the median and mean over the sites that have a value, and how many did */
|
|
115
|
+
export type Summary = {
|
|
116
|
+
median: number | null;
|
|
117
|
+
mean: number | null;
|
|
118
|
+
sites: number;
|
|
119
|
+
};
|
|
120
|
+
/** the churn between two readings of the same sites, over the sites both hold */
|
|
121
|
+
export type ChurnFigures = {
|
|
122
|
+
sites: ChurnSite[];
|
|
123
|
+
changedShare: Summary;
|
|
124
|
+
dictRatio: Summary;
|
|
125
|
+
/** sites whose aggregate names are identical on both readings */
|
|
126
|
+
unchangedSites: number;
|
|
127
|
+
};
|
|
128
|
+
/**
|
|
129
|
+
* Churn between an earlier and a later trim reading. Names are compared by path, so a changed
|
|
130
|
+
* name is an aggregate the later reading holds and the earlier one did not. The dictionary ratio
|
|
131
|
+
* uses the base sizes the later reading recorded; the earlier reading carries none.
|
|
132
|
+
*/
|
|
133
|
+
export declare function churnFigures(prior: readonly Row[], current: readonly Row[]): ChurnFigures;
|
|
78
134
|
/** coverage shares of the coverage reading (`measurements/trim/<date>.json`), pooled over its pages */
|
|
79
135
|
export declare function coverageFigures(rows: readonly Row[]): CoverageFigures & {
|
|
80
136
|
pages: number;
|