@decohere/ci-tools 0.0.1-dev.13 → 0.0.1-dev.131
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/package.json +17 -3
- package/src/audit/enforce.mjs +7 -5
- package/src/audit/format.mjs +133 -0
- package/src/audit/summarize.mjs +10 -32
- package/src/audit/upsert-issue.mjs +25 -74
- package/src/bench/bench.mjs +135 -368
- package/src/bench/file-regressions.mjs +76 -0
- package/src/bench/history.mjs +248 -0
- package/src/bench/nightly.mjs +118 -0
- package/src/bench/regression-issue.mjs +44 -0
- package/src/bench/trace.mjs +18 -14
- package/src/github/npm-bootstrap.mjs +179 -0
- package/src/github/oidc-bootstrap-nudge.mjs +214 -0
- package/src/github/publish-verify.mjs +206 -0
- package/src/github/smoke-create-launcher.mjs +85 -0
- package/src/lib/github-issue.mjs +75 -0
- package/src/lib/lcov.mjs +1 -1
- package/src/lib/workspace-packages.mjs +30 -0
- package/src/pages/artifact.mjs +38 -183
- package/src/pages/bench-page.mjs +324 -0
- package/src/pages/restore-badges.mjs +35 -0
- package/src/pages/restore-bench-history.mjs +15 -28
- package/src/release/npm-links.mjs +6 -13
- package/src/release/sync-dependencies.mjs +1 -2
- package/tests/audit/format.test.mjs +115 -0
- package/tests/bench/history.test.mjs +279 -0
- package/tests/bench/regression-issue.test.mjs +42 -0
- package/tests/github/ci-workflow.test.mjs +19 -0
- package/tests/github/npm-bootstrap.test.mjs +125 -0
- package/tests/github/oidc-bootstrap-nudge.test.mjs +89 -0
- package/tests/github/package-metadata.test.mjs +69 -0
- package/tests/github/publish-verify.test.mjs +115 -0
- package/tests/github/smoke-create-launcher.test.mjs +132 -0
- package/tests/lib/github-issue.test.mjs +85 -0
- package/tests/lib/lcov.test.mjs +8 -8
- package/tests/pages/bench-page.test.mjs +123 -0
- package/src/coverage/enforce-thresholds.mjs +0 -144
- package/src/pages/clear-bench-history.mjs +0 -27
- package/src/pages/keep-prior-bench-badge.mjs +0 -38
package/src/pages/artifact.mjs
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { cp, mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
2
3
|
import path from "node:path";
|
|
4
|
+
import { normalize } from "../bench/history.mjs";
|
|
5
|
+
import { buildBenchmarkHtml, buildStylesCss } from "./bench-page.mjs";
|
|
3
6
|
|
|
4
7
|
const branch = process.env.GITHUB_REF_NAME ?? "local";
|
|
5
8
|
const sha = process.env.GITHUB_SHA ?? "local";
|
|
@@ -8,201 +11,53 @@ const outputDir = process.env.PAGES_OUTPUT_DIR ?? "site";
|
|
|
8
11
|
const badgesDir = "projects/ci-tools/dist/ci/badges";
|
|
9
12
|
const benchHistoryPath = "projects/ci-tools/dist/ci/bench/bench-history.json";
|
|
10
13
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
function averageBenchmarks(runs) {
|
|
25
|
-
const first = runs[0]?.vitestOutput;
|
|
26
|
-
if (first == null) return [];
|
|
27
|
-
|
|
28
|
-
const latestBenchmarks = first.files
|
|
29
|
-
.flatMap((file) => file.groups)
|
|
30
|
-
.flatMap((group) => group.benchmarks);
|
|
31
|
-
|
|
32
|
-
return latestBenchmarks.map((benchmark) => {
|
|
33
|
-
const peers = runs
|
|
34
|
-
.flatMap((run) => run.vitestOutput?.files ?? [])
|
|
35
|
-
.flatMap((file) => file.groups ?? [])
|
|
36
|
-
.flatMap((group) => group.benchmarks ?? [])
|
|
37
|
-
.filter((candidate) => candidate.name === benchmark.name);
|
|
38
|
-
|
|
39
|
-
const averaged = { name: benchmark.name };
|
|
40
|
-
for (const field of numericFields) {
|
|
41
|
-
const values = peers
|
|
42
|
-
.map((peer) => peer[field])
|
|
43
|
-
.filter((value) => typeof value === "number");
|
|
44
|
-
if (values.length > 0) {
|
|
45
|
-
averaged[field] = values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
return averaged;
|
|
50
|
-
});
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
export function buildBenchmarkHtml(benchHistory) {
|
|
54
|
-
const contexts = Object.entries(benchHistory.runs ?? {})
|
|
55
|
-
.filter(([, entry]) => Array.isArray(entry.runs) && entry.runs.length > 0)
|
|
56
|
-
.sort((a, b) => new Date(b[1].runs[0].date) - new Date(a[1].runs[0].date));
|
|
57
|
-
|
|
58
|
-
if (contexts.length === 0) {
|
|
59
|
-
return `<!doctype html>
|
|
60
|
-
<html lang="en">
|
|
61
|
-
<head>
|
|
62
|
-
<meta charset="utf-8" />
|
|
63
|
-
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
64
|
-
<title>Ark - Benchmarks</title>
|
|
65
|
-
<link rel="stylesheet" href="./styles.css" />
|
|
66
|
-
</head>
|
|
67
|
-
<body>
|
|
68
|
-
<main class="container">
|
|
69
|
-
<header class="hero">
|
|
70
|
-
<h1>Benchmarks</h1>
|
|
71
|
-
<p class="note">No benchmark history is available yet.</p>
|
|
72
|
-
</header>
|
|
73
|
-
</main>
|
|
74
|
-
</body>
|
|
75
|
-
</html>
|
|
76
|
-
`;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
const rows = [];
|
|
80
|
-
for (const [, entry] of contexts) {
|
|
81
|
-
const benchmarks = averageBenchmarks(entry.runs);
|
|
82
|
-
const ctx = entry.context;
|
|
83
|
-
const environment = `${ctx.benchEnv ?? "unknown"}-${ctx.os.arch}`;
|
|
84
|
-
for (const benchmark of benchmarks) {
|
|
85
|
-
rows.push(`<tr>
|
|
86
|
-
<td>${environment}</td>
|
|
87
|
-
<td><code>${benchmark.name}</code></td>
|
|
88
|
-
<td>${fmtMs(benchmark.mean)}</td>
|
|
89
|
-
<td>${typeof benchmark.rme === "number" ? `±${benchmark.rme.toFixed(1)}%` : "-"}</td>
|
|
90
|
-
<td>${fmtMs(benchmark.p99)}</td>
|
|
91
|
-
<td>${fmtHz(benchmark.hz)}</td>
|
|
92
|
-
<td>${typeof benchmark.sampleCount === "number" ? Math.round(benchmark.sampleCount) : "-"}</td>
|
|
93
|
-
</tr>`);
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
const details = contexts
|
|
98
|
-
.map(([, entry]) => {
|
|
99
|
-
const ctx = entry.context;
|
|
100
|
-
return `<li><strong>${ctx.benchEnv ?? "unknown"}-${ctx.os.arch}</strong> - ${ctx.os.platform} - ${ctx.cpu.model} x ${ctx.cpu.count} - ${ctx.memoryGb} GB - Node ${ctx.node} - rolling mean of ${entry.runs.length} run(s)</li>`;
|
|
101
|
-
})
|
|
102
|
-
.join("\n");
|
|
103
|
-
|
|
104
|
-
const updatedDate = contexts[0][1].runs[0].date.slice(0, 10);
|
|
105
|
-
|
|
106
|
-
return `<!doctype html>
|
|
107
|
-
<html lang="en">
|
|
108
|
-
<head>
|
|
109
|
-
<meta charset="utf-8" />
|
|
110
|
-
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
111
|
-
<title>Ark - Benchmarks</title>
|
|
112
|
-
<link rel="stylesheet" href="./styles.css" />
|
|
113
|
-
</head>
|
|
114
|
-
<body>
|
|
115
|
-
<main class="container">
|
|
116
|
-
<div class="hero">
|
|
117
|
-
<h1>Benchmarks</h1>
|
|
118
|
-
<p class="note">Updated ${updatedDate} · Branch: ${branch} · Commit: ${sha.slice(0, 12)}</p>
|
|
119
|
-
</div>
|
|
120
|
-
<section class="section">
|
|
121
|
-
<h2>Performance Table</h2>
|
|
122
|
-
<table>
|
|
123
|
-
<thead>
|
|
124
|
-
<tr>
|
|
125
|
-
<th>Environment</th>
|
|
126
|
-
<th>Benchmark</th>
|
|
127
|
-
<th>Mean</th>
|
|
128
|
-
<th>±RME</th>
|
|
129
|
-
<th>P99</th>
|
|
130
|
-
<th>Ops/s</th>
|
|
131
|
-
<th>Runs</th>
|
|
132
|
-
</tr>
|
|
133
|
-
</thead>
|
|
134
|
-
<tbody>
|
|
135
|
-
${rows.join("\n")}
|
|
136
|
-
</tbody>
|
|
137
|
-
</table>
|
|
138
|
-
</section>
|
|
139
|
-
<section class="section">
|
|
140
|
-
<h2>Environment Notes</h2>
|
|
141
|
-
<ul>
|
|
142
|
-
${details}
|
|
143
|
-
</ul>
|
|
144
|
-
</section>
|
|
145
|
-
</main>
|
|
146
|
-
</body>
|
|
147
|
-
</html>
|
|
148
|
-
`;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
function buildStylesCss() {
|
|
152
|
-
return `:root {
|
|
153
|
-
--bg: #f4f7f3;
|
|
154
|
-
--surface: #ffffff;
|
|
155
|
-
--text: #132b24;
|
|
156
|
-
--muted: #4f6f65;
|
|
157
|
-
--accent: #0f9d72;
|
|
158
|
-
--accent-strong: #0d7f5c;
|
|
159
|
-
--border: #d6e3dd;
|
|
160
|
-
--shadow: 0 14px 40px rgba(12, 58, 45, 0.08);
|
|
161
|
-
}
|
|
162
|
-
* { box-sizing: border-box; }
|
|
163
|
-
body { margin: 0; font-family: "Segoe UI", "Helvetica Neue", sans-serif; color: var(--text); background: radial-gradient(circle at top right, #dbefe7, var(--bg) 40%); }
|
|
164
|
-
a { color: var(--accent-strong); }
|
|
165
|
-
.container { max-width: 1100px; margin: 0 auto; padding: 28px 18px 60px; }
|
|
166
|
-
.hero { background: linear-gradient(145deg, #183a31, #0e7f5c); color: #f3fff8; padding: 26px; border-radius: 18px; box-shadow: var(--shadow); }
|
|
167
|
-
.hero h1 { margin: 6px 0 10px; }
|
|
168
|
-
.note { color: #d8f5eb; margin: 0; }
|
|
169
|
-
.section { margin-top: 26px; }
|
|
170
|
-
table { width: 100%; border-collapse: collapse; background: var(--surface); border-radius: 12px; overflow: hidden; }
|
|
171
|
-
th, td { border-bottom: 1px solid var(--border); padding: 9px 10px; text-align: left; }
|
|
172
|
-
th { color: var(--muted); font-weight: 600; background: #f3faf7; }
|
|
173
|
-
td code { color: var(--accent-strong); }
|
|
174
|
-
@media (max-width: 700px) { .hero { padding: 20px; } .container { padding: 18px 12px 40px; } }
|
|
175
|
-
`;
|
|
14
|
+
/** Every badge file under `dir`, relative to it: the manifest restore-badges.mjs reads back. */
|
|
15
|
+
async function listBadges(dir, prefix = "") {
|
|
16
|
+
const entries = await readdir(path.join(dir, prefix), { withFileTypes: true });
|
|
17
|
+
const nested = await Promise.all(
|
|
18
|
+
entries.map((entry) =>
|
|
19
|
+
entry.isDirectory()
|
|
20
|
+
? listBadges(dir, path.posix.join(prefix, entry.name))
|
|
21
|
+
: entry.name.endsWith(".svg")
|
|
22
|
+
? [path.posix.join(prefix, entry.name)]
|
|
23
|
+
: [],
|
|
24
|
+
),
|
|
25
|
+
);
|
|
26
|
+
return nested.flat().sort();
|
|
176
27
|
}
|
|
177
28
|
|
|
178
29
|
await rm(outputDir, { recursive: true, force: true });
|
|
179
|
-
await mkdir(outputDir, { recursive: true });
|
|
30
|
+
await mkdir(path.join(outputDir, "data"), { recursive: true });
|
|
180
31
|
|
|
181
32
|
await writeFile(path.join(outputDir, "styles.css"), buildStylesCss());
|
|
182
33
|
|
|
183
34
|
// index.html redirects to benchmark.html — no separate dashboard needed
|
|
184
35
|
await writeFile(
|
|
185
36
|
path.join(outputDir, "index.html"),
|
|
186
|
-
`<!doctype html><html><head><meta charset="utf-8"/><meta http-equiv="refresh" content="0;url=./benchmark.html"/><title>
|
|
37
|
+
`<!doctype html><html><head><meta charset="utf-8"/><meta http-equiv="refresh" content="0;url=./benchmark.html"/><title>Decohere Pages</title></head><body><p><a href="./benchmark.html">Benchmarks</a></p></body></html>\n`,
|
|
187
38
|
);
|
|
188
39
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
await
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
40
|
+
// No history yet (first deploy, or cleared) publishes the empty page, not a failure.
|
|
41
|
+
const benchHistory = normalize(
|
|
42
|
+
existsSync(benchHistoryPath) ? JSON.parse(await readFile(benchHistoryPath, "utf8")) : null,
|
|
43
|
+
);
|
|
44
|
+
await writeFile(
|
|
45
|
+
path.join(outputDir, "benchmark.html"),
|
|
46
|
+
buildBenchmarkHtml(benchHistory, { branch, sha }),
|
|
47
|
+
);
|
|
48
|
+
await writeFile(
|
|
49
|
+
path.join(outputDir, "data", "bench-history.json"),
|
|
50
|
+
`${JSON.stringify(benchHistory, null, 2)}\n`,
|
|
51
|
+
);
|
|
201
52
|
|
|
202
|
-
|
|
53
|
+
// Badges are generated by earlier steps (CI status, coverage) or restored from the live site.
|
|
54
|
+
if (existsSync(badgesDir)) {
|
|
203
55
|
await cp(badgesDir, path.join(outputDir, "badges"), { recursive: true });
|
|
204
|
-
|
|
205
|
-
|
|
56
|
+
const badges = await listBadges(badgesDir);
|
|
57
|
+
await writeFile(
|
|
58
|
+
path.join(outputDir, "badges", "manifest.json"),
|
|
59
|
+
`${JSON.stringify({ badges }, null, 2)}\n`,
|
|
60
|
+
);
|
|
206
61
|
}
|
|
207
62
|
|
|
208
63
|
await writeFile(path.join(outputDir, ".nojekyll"), "");
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The benchmark page on Pages: a trend chart per benchmark and a table of the latest
|
|
3
|
+
* results, from the version 2 history (../bench/history.mjs). Static HTML and inline SVG, so the
|
|
4
|
+
* page needs no script and no third-party chart library.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { REGRESSION, STALE_AFTER_DAYS } from "../bench/history.mjs";
|
|
8
|
+
|
|
9
|
+
const GROUPS = [
|
|
10
|
+
{ prefix: "flux.serve.", title: "flux serve (real process)" },
|
|
11
|
+
{ prefix: "flux.run.", title: "flux run (installed CLI, real process)" },
|
|
12
|
+
{ prefix: "flux.load.", title: "flux load (in process)" },
|
|
13
|
+
{ prefix: "flux.transpile.", title: "Transpile strategies (SWC only)" },
|
|
14
|
+
{ prefix: "forge.build.", title: "forge build" },
|
|
15
|
+
];
|
|
16
|
+
|
|
17
|
+
const SERIES_COLORS = [
|
|
18
|
+
"var(--series-1)",
|
|
19
|
+
"var(--series-2)",
|
|
20
|
+
"var(--series-3)",
|
|
21
|
+
"var(--series-4)",
|
|
22
|
+
];
|
|
23
|
+
|
|
24
|
+
function escapeHtml(value) {
|
|
25
|
+
return String(value)
|
|
26
|
+
.replaceAll("&", "&")
|
|
27
|
+
.replaceAll("<", "<")
|
|
28
|
+
.replaceAll(">", ">")
|
|
29
|
+
.replaceAll('"', """);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function fmtMs(ms) {
|
|
33
|
+
if (typeof ms !== "number" || Number.isNaN(ms)) return "–";
|
|
34
|
+
if (ms === 0) return "0 ms";
|
|
35
|
+
if (ms >= 100) return `${Math.round(ms)} ms`;
|
|
36
|
+
return `${ms.toFixed(1)} ms`;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** `2026-10-20 03:17 UTC`: when a run happened, to the minute. */
|
|
40
|
+
function fmtDate(iso) {
|
|
41
|
+
return `${iso.slice(0, 10)} ${iso.slice(11, 16)} UTC`;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function commitLink(repoUrl, sha) {
|
|
45
|
+
if (sha == null) return "–";
|
|
46
|
+
return `<a href="${escapeHtml(`${repoUrl}/commit/${sha}`)}"><code>${escapeHtml(sha.slice(0, 7))}</code></a>`;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function machineName(context) {
|
|
50
|
+
return `${context.benchEnv ?? "unknown"} · ${context.cpu.model} × ${context.cpu.count}`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Contexts with a run within STALE_AFTER_DAYS of the newest run anywhere, most recently run first:
|
|
55
|
+
* CI runner types come and go, and one that stopped appearing would otherwise keep a line.
|
|
56
|
+
*/
|
|
57
|
+
function activeContexts(history) {
|
|
58
|
+
const newest = Math.max(
|
|
59
|
+
...Object.values(history.contexts ?? {})
|
|
60
|
+
.filter((entry) => entry.runs.length > 0)
|
|
61
|
+
.map((entry) => new Date(entry.runs[0].date).getTime()),
|
|
62
|
+
);
|
|
63
|
+
const cutoff = newest - STALE_AFTER_DAYS * 24 * 60 * 60 * 1000;
|
|
64
|
+
return Object.entries(history.contexts ?? {})
|
|
65
|
+
.filter(([, entry]) => entry.runs.length > 0)
|
|
66
|
+
.filter(([, entry]) => new Date(entry.runs[0].date).getTime() >= cutoff)
|
|
67
|
+
.sort((a, b) => new Date(b[1].runs[0].date) - new Date(a[1].runs[0].date))
|
|
68
|
+
.map(([id, entry], index) => ({
|
|
69
|
+
id,
|
|
70
|
+
context: entry.context,
|
|
71
|
+
runs: entry.runs,
|
|
72
|
+
color: SERIES_COLORS[index % SERIES_COLORS.length],
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Every benchmark id seen in any run, with the label from its newest run on any machine. */
|
|
77
|
+
function benchmarkIds(contexts) {
|
|
78
|
+
const newest = new Map();
|
|
79
|
+
for (const { runs } of contexts) {
|
|
80
|
+
for (const run of runs) {
|
|
81
|
+
const time = new Date(run.date).getTime();
|
|
82
|
+
for (const result of run.results) {
|
|
83
|
+
const seen = newest.get(result.id);
|
|
84
|
+
if (seen == null || time > seen.time)
|
|
85
|
+
newest.set(result.id, { time, label: result.label });
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return new Map([...newest].map(([id, { label }]) => [id, label]));
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Oldest-first points for one benchmark on one machine. */
|
|
93
|
+
function seriesFor(runs, id) {
|
|
94
|
+
return [...runs]
|
|
95
|
+
.reverse()
|
|
96
|
+
.map((run) => ({ run, result: run.results.find((result) => result.id === id) }))
|
|
97
|
+
.filter((point) => point.result != null)
|
|
98
|
+
.map(({ run, result }) => ({
|
|
99
|
+
time: new Date(run.date).getTime(),
|
|
100
|
+
date: run.date.slice(0, 10),
|
|
101
|
+
iso: run.date,
|
|
102
|
+
sha: run.sha,
|
|
103
|
+
median: result.median,
|
|
104
|
+
p90: result.p90,
|
|
105
|
+
}));
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** A line chart of medians over time, one line per machine, with a hover title per point. */
|
|
109
|
+
export function trendSvg(lines) {
|
|
110
|
+
const width = 560;
|
|
111
|
+
const height = 180;
|
|
112
|
+
const pad = { top: 12, right: 12, bottom: 24, left: 56 };
|
|
113
|
+
const points = lines.flatMap((line) => line.points);
|
|
114
|
+
if (points.length === 0) return "";
|
|
115
|
+
const minTime = Math.min(...points.map((p) => p.time));
|
|
116
|
+
const maxTime = Math.max(...points.map((p) => p.time));
|
|
117
|
+
const maxValue = Math.max(...points.map((p) => p.p90 ?? p.median)) * 1.1 || 1;
|
|
118
|
+
const x = (time) =>
|
|
119
|
+
maxTime === minTime
|
|
120
|
+
? pad.left + (width - pad.left - pad.right) / 2
|
|
121
|
+
: pad.left + ((time - minTime) / (maxTime - minTime)) * (width - pad.left - pad.right);
|
|
122
|
+
const y = (value) => pad.top + (1 - value / maxValue) * (height - pad.top - pad.bottom);
|
|
123
|
+
const ticks = [0, 0.5, 1].map((fraction) => maxValue * fraction);
|
|
124
|
+
const grid = ticks
|
|
125
|
+
.map(
|
|
126
|
+
(value) =>
|
|
127
|
+
`<line class="grid" x1="${pad.left}" x2="${width - pad.right}" y1="${y(value)}" y2="${y(value)}"/>` +
|
|
128
|
+
`<text class="axis" x="${pad.left - 6}" y="${y(value) + 4}" text-anchor="end">${fmtMs(value)}</text>`,
|
|
129
|
+
)
|
|
130
|
+
.join("");
|
|
131
|
+
const dateLabels = [
|
|
132
|
+
{ time: minTime, anchor: "start" },
|
|
133
|
+
...(maxTime !== minTime ? [{ time: maxTime, anchor: "end" }] : []),
|
|
134
|
+
]
|
|
135
|
+
.map(
|
|
136
|
+
({ time, anchor }) =>
|
|
137
|
+
`<text class="axis" x="${x(time)}" y="${height - 6}" text-anchor="${anchor}">${new Date(time).toISOString().slice(0, 10)}</text>`,
|
|
138
|
+
)
|
|
139
|
+
.join("");
|
|
140
|
+
const series = lines
|
|
141
|
+
.map((line) => {
|
|
142
|
+
const path = line.points
|
|
143
|
+
.map((p) => `${x(p.time).toFixed(1)},${y(p.median).toFixed(1)}`)
|
|
144
|
+
.join(" ");
|
|
145
|
+
const dots = line.points
|
|
146
|
+
.map(
|
|
147
|
+
(p) =>
|
|
148
|
+
`<circle cx="${x(p.time).toFixed(1)}" cy="${y(p.median).toFixed(1)}" r="3" style="fill:${line.color}"><title>${escapeHtml(
|
|
149
|
+
`${p.date}${p.sha ? ` · ${p.sha.slice(0, 7)}` : ""} · median ${fmtMs(p.median)}${p.p90 != null ? ` · p90 ${fmtMs(p.p90)}` : ""}`,
|
|
150
|
+
)}</title></circle>`,
|
|
151
|
+
)
|
|
152
|
+
.join("");
|
|
153
|
+
return `<polyline points="${path}" style="stroke:${line.color}"/>${dots}`;
|
|
154
|
+
})
|
|
155
|
+
.join("");
|
|
156
|
+
return `<svg class="trend" viewBox="0 0 ${width} ${height}" role="img" aria-label="Median over time">${grid}${dateLabels}${series}</svg>`;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function changeText(points) {
|
|
160
|
+
const last = points.at(-1);
|
|
161
|
+
const before = points.at(-2);
|
|
162
|
+
if (last == null || before == null || before.median === 0) return "";
|
|
163
|
+
const percent = ((last.median - before.median) / before.median) * 100;
|
|
164
|
+
const direction = percent > 0 ? "slower" : "faster";
|
|
165
|
+
if (Math.abs(percent) < 0.5) return `<span class="change">no change</span>`;
|
|
166
|
+
return `<span class="change ${percent > 0 ? "worse" : "better"}">${Math.abs(percent).toFixed(1)}% ${direction}</span>`;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function card(id, label, contexts) {
|
|
170
|
+
const lines = contexts
|
|
171
|
+
.map((ctx) => ({
|
|
172
|
+
color: ctx.color,
|
|
173
|
+
name: machineName(ctx.context),
|
|
174
|
+
points: seriesFor(ctx.runs, id),
|
|
175
|
+
}))
|
|
176
|
+
.filter((line) => line.points.length > 0);
|
|
177
|
+
const primary = lines[0];
|
|
178
|
+
const latest = primary?.points.at(-1);
|
|
179
|
+
return `<article class="card">
|
|
180
|
+
<header>
|
|
181
|
+
<h3>${escapeHtml(label)}</h3>
|
|
182
|
+
<code>${escapeHtml(id)}</code>
|
|
183
|
+
</header>
|
|
184
|
+
<p class="latest"><strong>${fmtMs(latest?.median)}</strong> median${latest?.p90 != null ? ` · p90 ${fmtMs(latest.p90)}` : ""} ${primary ? changeText(primary.points) : ""}</p>
|
|
185
|
+
<p class="when">${latest != null ? `Last run ${escapeHtml(fmtDate(latest.iso))}` : ""}</p>
|
|
186
|
+
${trendSvg(lines)}
|
|
187
|
+
</article>`;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function latestTable(contexts, labels, repoUrl) {
|
|
191
|
+
const rows = contexts.flatMap((ctx) =>
|
|
192
|
+
ctx.runs[0].results.map(
|
|
193
|
+
(result) => `<tr>
|
|
194
|
+
<td><span class="swatch" style="background:${ctx.color}"></span>${escapeHtml(ctx.context.benchEnv ?? "unknown")}</td>
|
|
195
|
+
<td>${escapeHtml(fmtDate(ctx.runs[0].date))}</td>
|
|
196
|
+
<td>${commitLink(repoUrl, ctx.runs[0].sha)}</td>
|
|
197
|
+
<td>${escapeHtml(labels.get(result.id) ?? result.label)}</td>
|
|
198
|
+
<td class="num">${fmtMs(result.median)}</td>
|
|
199
|
+
<td class="num">${fmtMs(result.p90)}</td>
|
|
200
|
+
<td class="num">${fmtMs(result.min)}</td>
|
|
201
|
+
<td class="num">${result.files ?? "–"}</td>
|
|
202
|
+
<td class="num">${result.samples}</td>
|
|
203
|
+
</tr>`,
|
|
204
|
+
),
|
|
205
|
+
);
|
|
206
|
+
return `<div class="table-wrap"><table>
|
|
207
|
+
<thead><tr><th>Machine</th><th>Run</th><th>Commit</th><th>Benchmark</th><th class="num">Median</th><th class="num">p90</th><th class="num">Min</th><th class="num">Files</th><th class="num">Samples</th></tr></thead>
|
|
208
|
+
<tbody>${rows.join("\n")}</tbody>
|
|
209
|
+
</table></div>`;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
export function buildBenchmarkHtml(
|
|
213
|
+
history,
|
|
214
|
+
{ branch = "local", sha = "local", repoUrl = "https://github.com/typesmiths/decohere" } = {},
|
|
215
|
+
) {
|
|
216
|
+
const contexts = activeContexts(history);
|
|
217
|
+
const body =
|
|
218
|
+
contexts.length === 0
|
|
219
|
+
? `<p class="empty">No benchmark history yet. The nightly bench workflow adds a run each night.</p>`
|
|
220
|
+
: (() => {
|
|
221
|
+
const labels = benchmarkIds(contexts);
|
|
222
|
+
const grouped = GROUPS.map((group) => ({
|
|
223
|
+
...group,
|
|
224
|
+
ids: [...labels.keys()].filter((id) => id.startsWith(group.prefix)),
|
|
225
|
+
}));
|
|
226
|
+
const other = [...labels.keys()].filter(
|
|
227
|
+
(id) => !GROUPS.some((group) => id.startsWith(group.prefix)),
|
|
228
|
+
);
|
|
229
|
+
if (other.length > 0) grouped.push({ title: "Other", ids: other });
|
|
230
|
+
const sections = grouped
|
|
231
|
+
.filter((group) => group.ids.length > 0)
|
|
232
|
+
.map(
|
|
233
|
+
(group) => `<section>
|
|
234
|
+
<h2>${escapeHtml(group.title)}</h2>
|
|
235
|
+
<div class="cards">${group.ids.map((id) => card(id, labels.get(id), contexts)).join("\n")}</div>
|
|
236
|
+
</section>`,
|
|
237
|
+
)
|
|
238
|
+
.join("\n");
|
|
239
|
+
const legend = contexts
|
|
240
|
+
.map(
|
|
241
|
+
(ctx) =>
|
|
242
|
+
`<li><span class="swatch" style="background:${ctx.color}"></span>${escapeHtml(machineName(ctx.context))} · ${escapeHtml(ctx.context.os.platform)} ${escapeHtml(ctx.context.os.arch)} · ${ctx.context.memoryGb} GB · Node ${escapeHtml(ctx.context.node)} · ${ctx.runs.length} run(s)</li>`,
|
|
243
|
+
)
|
|
244
|
+
.join("\n");
|
|
245
|
+
return `<section class="machines"><h2>Machines</h2><ul>${legend}</ul>
|
|
246
|
+
<p class="note">Each line is one machine type; runs are only compared on the same one. Shared CI runners are noisy, so read the trend, not a single point. Hover a point for its date, commit and p90. Each night also benchmarks the previous nightly's commit on the same runner, alternating ${REGRESSION.rounds} rounds each; a benchmark more than ${Math.round(REGRESSION.threshold * 100)}% slower in every round gets an issue: <a href="${escapeHtml(`${repoUrl}/issues?q=is%3Aissue+is%3Aopen+%22perf(bench)%22`)}">open benchmark issues</a>.</p></section>
|
|
247
|
+
${sections}
|
|
248
|
+
<section><h2>Latest run</h2>${latestTable(contexts, labels, repoUrl)}</section>`;
|
|
249
|
+
})();
|
|
250
|
+
const updated = contexts[0]?.runs[0].date.slice(0, 10) ?? "never";
|
|
251
|
+
return `<!doctype html>
|
|
252
|
+
<html lang="en">
|
|
253
|
+
<head>
|
|
254
|
+
<meta charset="utf-8" />
|
|
255
|
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
256
|
+
<title>Decohere Benchmarks</title>
|
|
257
|
+
<link rel="stylesheet" href="./styles.css" />
|
|
258
|
+
</head>
|
|
259
|
+
<body>
|
|
260
|
+
<main class="container">
|
|
261
|
+
<header class="hero">
|
|
262
|
+
<h1>Decohere benchmarks</h1>
|
|
263
|
+
<p>Updated ${escapeHtml(updated)} · built from ${escapeHtml(branch)} @ <code>${escapeHtml(sha.slice(0, 12))}</code> · <a href="./data/bench-history.json">raw data</a></p>
|
|
264
|
+
</header>
|
|
265
|
+
${body}
|
|
266
|
+
</main>
|
|
267
|
+
</body>
|
|
268
|
+
</html>
|
|
269
|
+
`;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
export function buildStylesCss() {
|
|
273
|
+
return `:root {
|
|
274
|
+
--bg: #f6f7f9; --surface: #ffffff; --text: #1b1f24; --muted: #59636e; --border: #d8dee4;
|
|
275
|
+
--accent: #0b6e4f; --better: #1a7f37; --worse: #cf222e;
|
|
276
|
+
--series-1: #0b6e4f; --series-2: #8250df; --series-3: #bc4c00; --series-4: #0969da;
|
|
277
|
+
color-scheme: light dark;
|
|
278
|
+
}
|
|
279
|
+
@media (prefers-color-scheme: dark) {
|
|
280
|
+
:root:not([data-theme="light"]) {
|
|
281
|
+
--bg: #0d1117; --surface: #161b22; --text: #e6edf3; --muted: #9198a1; --border: #30363d;
|
|
282
|
+
--accent: #3fb68b; --better: #3fb950; --worse: #f85149;
|
|
283
|
+
--series-1: #3fb68b; --series-2: #a371f7; --series-3: #f0883e; --series-4: #4493f8;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
:root[data-theme="dark"] {
|
|
287
|
+
--bg: #0d1117; --surface: #161b22; --text: #e6edf3; --muted: #9198a1; --border: #30363d;
|
|
288
|
+
--accent: #3fb68b; --better: #3fb950; --worse: #f85149;
|
|
289
|
+
--series-1: #3fb68b; --series-2: #a371f7; --series-3: #f0883e; --series-4: #4493f8;
|
|
290
|
+
}
|
|
291
|
+
* { box-sizing: border-box; }
|
|
292
|
+
body { margin: 0; background: var(--bg); color: var(--text); font: 15px/1.5 system-ui, -apple-system, "Segoe UI", sans-serif; }
|
|
293
|
+
a { color: var(--accent); }
|
|
294
|
+
code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.85em; }
|
|
295
|
+
.container { max-width: 1180px; margin: 0 auto; padding: 24px 16px 64px; }
|
|
296
|
+
.hero h1 { margin: 0 0 4px; font-size: 1.6rem; }
|
|
297
|
+
.hero p, .note, .empty { color: var(--muted); margin: 0; }
|
|
298
|
+
section { margin-top: 32px; }
|
|
299
|
+
h2 { font-size: 1.1rem; margin: 0 0 12px; }
|
|
300
|
+
.machines ul { list-style: none; padding: 0; margin: 0 0 8px; }
|
|
301
|
+
.machines li { margin: 4px 0; }
|
|
302
|
+
.swatch { display: inline-block; width: 10px; height: 10px; border-radius: 2px; margin-right: 8px; vertical-align: baseline; }
|
|
303
|
+
.cards { display: grid; grid-template-columns: repeat(auto-fill, minmax(min(100%, 360px), 1fr)); gap: 16px; }
|
|
304
|
+
.card { background: var(--surface); border: 1px solid var(--border); border-radius: 10px; padding: 14px 16px; min-width: 0; }
|
|
305
|
+
.card header { display: flex; flex-direction: column; gap: 2px; }
|
|
306
|
+
.card h3 { font-size: 0.95rem; margin: 0; }
|
|
307
|
+
.card header code { color: var(--muted); }
|
|
308
|
+
.latest { margin: 8px 0 0; }
|
|
309
|
+
.when { margin: 0 0 4px; color: var(--muted); font-size: 0.85rem; }
|
|
310
|
+
.latest strong { font-size: 1.25rem; font-variant-numeric: tabular-nums; }
|
|
311
|
+
.change { margin-left: 6px; font-size: 0.85rem; color: var(--muted); }
|
|
312
|
+
.change.better { color: var(--better); }
|
|
313
|
+
.change.worse { color: var(--worse); }
|
|
314
|
+
.trend { width: 100%; height: auto; display: block; }
|
|
315
|
+
.trend polyline { fill: none; stroke-width: 2; }
|
|
316
|
+
.trend .grid { stroke: var(--border); stroke-width: 1; }
|
|
317
|
+
.trend .axis { fill: var(--muted); font-size: 11px; }
|
|
318
|
+
.table-wrap { overflow-x: auto; }
|
|
319
|
+
table { width: 100%; border-collapse: collapse; background: var(--surface); border: 1px solid var(--border); border-radius: 10px; font-variant-numeric: tabular-nums; }
|
|
320
|
+
th, td { padding: 8px 10px; border-bottom: 1px solid var(--border); text-align: left; white-space: nowrap; }
|
|
321
|
+
th { color: var(--muted); font-weight: 600; }
|
|
322
|
+
.num { text-align: right; }
|
|
323
|
+
`;
|
|
324
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// Restore the badges on the live Pages site into dist, so a deploy that does not regenerate
|
|
4
|
+
// them (the nightly bench) keeps them. Pages replaces the whole site on every deploy.
|
|
5
|
+
//
|
|
6
|
+
// The site lists its badges in badges/manifest.json (artifact.mjs). A site with no manifest yet
|
|
7
|
+
// has no badges to keep. A badge the site does not have (404) is skipped; any other failure
|
|
8
|
+
// stops the job.
|
|
9
|
+
|
|
10
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
|
|
13
|
+
const BADGES_DIR = "projects/ci-tools/dist/ci/badges";
|
|
14
|
+
const PAGES_URL = process.env.PAGES_URL ?? "https://typesmiths.github.io/decohere";
|
|
15
|
+
|
|
16
|
+
async function fetchOrNull(url) {
|
|
17
|
+
const response = await fetch(url);
|
|
18
|
+
if (response.status === 404) return null;
|
|
19
|
+
if (!response.ok) throw new Error(`GET ${url}: HTTP ${response.status}`);
|
|
20
|
+
return response;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const manifest = await fetchOrNull(`${PAGES_URL}/badges/manifest.json`);
|
|
24
|
+
const badges = manifest != null ? (await manifest.json()).badges : [];
|
|
25
|
+
|
|
26
|
+
let restored = 0;
|
|
27
|
+
for (const badge of badges) {
|
|
28
|
+
const response = await fetchOrNull(`${PAGES_URL}/badges/${badge}`);
|
|
29
|
+
if (response == null) continue;
|
|
30
|
+
const file = path.join(BADGES_DIR, badge);
|
|
31
|
+
await mkdir(path.dirname(file), { recursive: true });
|
|
32
|
+
await writeFile(file, await response.text());
|
|
33
|
+
restored += 1;
|
|
34
|
+
}
|
|
35
|
+
console.log(`✓ Restored ${restored} badge(s) from Pages`);
|
|
@@ -1,37 +1,24 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
-
// Restore benchmark history from Pages, or
|
|
3
|
+
// Restore benchmark history from Pages. Only a 404 (never published, or cleared) starts an
|
|
4
|
+
// empty history: any other failure stops the job, so a network blip can't wipe the trend on
|
|
5
|
+
// the next save.
|
|
4
6
|
|
|
5
|
-
import {
|
|
6
|
-
import { writeFileSync } from "node:fs";
|
|
7
|
-
import { mkdir } from "node:fs/promises";
|
|
7
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
8
8
|
|
|
9
9
|
const HISTORY_DIR = "projects/ci-tools/dist/ci/bench";
|
|
10
10
|
const HISTORY_FILE = `${HISTORY_DIR}/bench-history.json`;
|
|
11
|
-
const PAGES_URL = "https://typesmiths.github.io/decohere
|
|
11
|
+
const PAGES_URL = process.env.PAGES_URL ?? "https://typesmiths.github.io/decohere";
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
console.log("✓ Restored benchmark history from Pages");
|
|
21
|
-
} catch {
|
|
22
|
-
console.log("No benchmark history on Pages; starting from empty history");
|
|
23
|
-
writeFileSync(
|
|
24
|
-
HISTORY_FILE,
|
|
25
|
-
JSON.stringify(
|
|
26
|
-
{
|
|
27
|
-
runs: {},
|
|
28
|
-
},
|
|
29
|
-
null,
|
|
30
|
-
2,
|
|
31
|
-
),
|
|
32
|
-
);
|
|
33
|
-
}
|
|
34
|
-
} catch (error) {
|
|
35
|
-
console.error("Error restoring benchmark history:", error.message);
|
|
13
|
+
await mkdir(HISTORY_DIR, { recursive: true });
|
|
14
|
+
const response = await fetch(`${PAGES_URL}/data/bench-history.json`);
|
|
15
|
+
if (response.status === 404) {
|
|
16
|
+
console.log("No benchmark history on Pages; starting from empty history");
|
|
17
|
+
await writeFile(HISTORY_FILE, `${JSON.stringify({ version: 2, contexts: {} }, null, 2)}\n`);
|
|
18
|
+
} else if (!response.ok) {
|
|
19
|
+
console.error(`Failed to restore benchmark history: HTTP ${response.status}`);
|
|
36
20
|
process.exit(1);
|
|
21
|
+
} else {
|
|
22
|
+
await writeFile(HISTORY_FILE, await response.text());
|
|
23
|
+
console.log("✓ Restored benchmark history from Pages");
|
|
37
24
|
}
|