@decohere/ci-tools 0.0.1-dev.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +41 -0
- package/src/audit/enforce.mjs +22 -0
- package/src/audit/summarize.mjs +46 -0
- package/src/audit/upsert-issue.mjs +90 -0
- package/src/badges/ci-status.mjs +52 -0
- package/src/bench/bench.mjs +411 -0
- package/src/bench/trace.mjs +183 -0
- package/src/changeset/aggregate-changelog.mjs +119 -0
- package/src/changeset/check-status.mjs +54 -0
- package/src/coverage/badge.mjs +57 -0
- package/src/coverage/enforce-thresholds.mjs +144 -0
- package/src/coverage/report.mjs +138 -0
- package/src/github/create-issue.mjs +173 -0
- package/src/github/create-or-update-pr.mjs +143 -0
- package/src/github/ensure-metadata.mjs +295 -0
- package/src/lib/lcov.mjs +127 -0
- package/src/lib/svg.mjs +66 -0
- package/src/pages/artifact.mjs +208 -0
- package/src/pages/clear-bench-history.mjs +27 -0
- package/src/pages/keep-prior-bench-badge.mjs +38 -0
- package/src/pages/restore-bench-history.mjs +37 -0
- package/src/pages/smoke.mjs +37 -0
- package/src/release/capture-version.mjs +27 -0
- package/src/release/npm-links.mjs +39 -0
- package/src/release/push-prerelease-commit.mjs +85 -0
- package/src/release/sync-dependencies.mjs +27 -0
- package/tests/lib/lcov.test.mjs +84 -0
- package/tests/lib/svg.test.mjs +94 -0
package/package.json
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@decohere/ci-tools",
|
|
3
|
+
"version": "0.0.1-dev.10",
|
|
4
|
+
"description": "CI utility scripts for badge generation, coverage reporting, pages artifact assembly, and GitHub API tooling",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"typescript",
|
|
7
|
+
"ci",
|
|
8
|
+
"badges",
|
|
9
|
+
"coverage",
|
|
10
|
+
"github-actions",
|
|
11
|
+
"decohere"
|
|
12
|
+
],
|
|
13
|
+
"license": "ISC",
|
|
14
|
+
"author": "Typesmiths",
|
|
15
|
+
"type": "module",
|
|
16
|
+
"exports": {
|
|
17
|
+
"./lib/svg": "./src/lib/svg.mjs",
|
|
18
|
+
"./lib/lcov": "./src/lib/lcov.mjs",
|
|
19
|
+
"./coverage/badge": "./src/coverage/badge.mjs",
|
|
20
|
+
"./coverage/report": "./src/coverage/report.mjs",
|
|
21
|
+
"./coverage/enforce-thresholds": "./src/coverage/enforce-thresholds.mjs",
|
|
22
|
+
"./badges/ci-status": "./src/badges/ci-status.mjs",
|
|
23
|
+
"./pages/artifact": "./src/pages/artifact.mjs",
|
|
24
|
+
"./pages/smoke": "./src/pages/smoke.mjs",
|
|
25
|
+
"./audit/summarize": "./src/audit/summarize.mjs",
|
|
26
|
+
"./audit/enforce": "./src/audit/enforce.mjs",
|
|
27
|
+
"./audit/upsert-issue": "./src/audit/upsert-issue.mjs",
|
|
28
|
+
"./bench/bench": "./src/bench/bench.mjs",
|
|
29
|
+
"./bench/trace": "./src/bench/trace.mjs",
|
|
30
|
+
"./changeset/aggregate-changelog": "./src/changeset/aggregate-changelog.mjs",
|
|
31
|
+
"./changeset/check-status": "./src/changeset/check-status.mjs",
|
|
32
|
+
"./github/create-issue": "./src/github/create-issue.mjs",
|
|
33
|
+
"./github/ensure-metadata": "./src/github/ensure-metadata.mjs",
|
|
34
|
+
"./github/create-or-update-pr": "./src/github/create-or-update-pr.mjs"
|
|
35
|
+
},
|
|
36
|
+
"scripts": {
|
|
37
|
+
"test": "vitest run"
|
|
38
|
+
},
|
|
39
|
+
"dependencies": {},
|
|
40
|
+
"devDependencies": {}
|
|
41
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
|
|
3
|
+
const auditFilePath = process.argv[2] ?? "npm-audit.json";
|
|
4
|
+
|
|
5
|
+
let data = null;
|
|
6
|
+
try {
|
|
7
|
+
data = JSON.parse(readFileSync(auditFilePath, "utf8"));
|
|
8
|
+
} catch {
|
|
9
|
+
console.error(`Security gate failed: could not parse ${auditFilePath}.`);
|
|
10
|
+
process.exit(1);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const vulnerabilities = data.metadata?.vulnerabilities ?? {};
|
|
14
|
+
const high = vulnerabilities.high ?? 0;
|
|
15
|
+
const critical = vulnerabilities.critical ?? 0;
|
|
16
|
+
|
|
17
|
+
if (high > 0 || critical > 0) {
|
|
18
|
+
console.error(`Security gate failed: high=${high}, critical=${critical}`);
|
|
19
|
+
process.exit(1);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
console.log("Security gate passed: no high/critical vulnerabilities found.");
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { appendFileSync, readFileSync } from "node:fs";
|
|
2
|
+
|
|
3
|
+
const auditFilePath = process.argv[2] ?? "npm-audit.json";
|
|
4
|
+
|
|
5
|
+
function writeSummary(lines) {
|
|
6
|
+
const summaryText = `${lines.join("\n")}\n`;
|
|
7
|
+
if (process.env.GITHUB_STEP_SUMMARY) {
|
|
8
|
+
appendFileSync(process.env.GITHUB_STEP_SUMMARY, summaryText);
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
process.stdout.write(summaryText);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
let data = null;
|
|
15
|
+
try {
|
|
16
|
+
data = JSON.parse(readFileSync(auditFilePath, "utf8"));
|
|
17
|
+
} catch {
|
|
18
|
+
writeSummary([
|
|
19
|
+
"## Security Audit",
|
|
20
|
+
"",
|
|
21
|
+
"- npm audit output could not be parsed.",
|
|
22
|
+
`- See uploaded \`${auditFilePath}\` artifact for raw output.`,
|
|
23
|
+
]);
|
|
24
|
+
process.exit(0);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const vulnerabilities = data.metadata?.vulnerabilities ?? {};
|
|
28
|
+
const info = vulnerabilities.info ?? 0;
|
|
29
|
+
const low = vulnerabilities.low ?? 0;
|
|
30
|
+
const moderate = vulnerabilities.moderate ?? 0;
|
|
31
|
+
const high = vulnerabilities.high ?? 0;
|
|
32
|
+
const critical = vulnerabilities.critical ?? 0;
|
|
33
|
+
const total = vulnerabilities.total ?? info + low + moderate + high + critical;
|
|
34
|
+
|
|
35
|
+
writeSummary([
|
|
36
|
+
"## Security Audit",
|
|
37
|
+
"",
|
|
38
|
+
`| Severity | Count |`,
|
|
39
|
+
`|-----------|-------|`,
|
|
40
|
+
`| Critical | ${critical} |`,
|
|
41
|
+
`| High | ${high} |`,
|
|
42
|
+
`| Moderate | ${moderate} |`,
|
|
43
|
+
`| Low | ${low} |`,
|
|
44
|
+
`| Info | ${info} |`,
|
|
45
|
+
`| **Total** | **${total}** |`,
|
|
46
|
+
]);
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
|
|
6
|
+
const auditFilePath = process.argv[2] ?? "npm-audit.json";
|
|
7
|
+
const ISSUE_TITLE = "security(ci): npm audit vulnerabilities detected";
|
|
8
|
+
|
|
9
|
+
function runGh(args) {
|
|
10
|
+
const result = spawnSync("gh", args, { encoding: "utf8", env: process.env });
|
|
11
|
+
if (result.status !== 0) {
|
|
12
|
+
throw new Error(result.stderr || result.stdout || `gh command failed: ${args.join(" ")}`);
|
|
13
|
+
}
|
|
14
|
+
return result.stdout;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function readAuditSummary(filePath) {
|
|
18
|
+
const data = JSON.parse(readFileSync(filePath, "utf8"));
|
|
19
|
+
const vulnerabilities = data.metadata?.vulnerabilities ?? {};
|
|
20
|
+
const info = vulnerabilities.info ?? 0;
|
|
21
|
+
const low = vulnerabilities.low ?? 0;
|
|
22
|
+
const moderate = vulnerabilities.moderate ?? 0;
|
|
23
|
+
const high = vulnerabilities.high ?? 0;
|
|
24
|
+
const critical = vulnerabilities.critical ?? 0;
|
|
25
|
+
const total = vulnerabilities.total ?? info + low + moderate + high + critical;
|
|
26
|
+
return { total, critical, high, moderate, low, info };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function buildBody(summary) {
|
|
30
|
+
const runUrl =
|
|
31
|
+
process.env.GITHUB_SERVER_URL && process.env.GITHUB_REPOSITORY && process.env.GITHUB_RUN_ID
|
|
32
|
+
? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
|
|
33
|
+
: "(run URL unavailable)";
|
|
34
|
+
const sha = process.env.GITHUB_SHA ? process.env.GITHUB_SHA.slice(0, 12) : "unknown";
|
|
35
|
+
|
|
36
|
+
return [
|
|
37
|
+
"## npm audit detected vulnerabilities",
|
|
38
|
+
"",
|
|
39
|
+
`- Total: ${summary.total}`,
|
|
40
|
+
`- Critical: ${summary.critical}`,
|
|
41
|
+
`- High: ${summary.high}`,
|
|
42
|
+
`- Moderate: ${summary.moderate}`,
|
|
43
|
+
`- Low: ${summary.low}`,
|
|
44
|
+
`- Info: ${summary.info}`,
|
|
45
|
+
"",
|
|
46
|
+
`- Commit: ${sha}`,
|
|
47
|
+
`- Workflow run: ${runUrl}`,
|
|
48
|
+
"",
|
|
49
|
+
"This issue is managed by CI and will be updated on subsequent main-branch runs while vulnerabilities remain.",
|
|
50
|
+
].join("\n");
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function findExistingIssueNumber() {
|
|
54
|
+
const output = runGh([
|
|
55
|
+
"issue",
|
|
56
|
+
"list",
|
|
57
|
+
"--state",
|
|
58
|
+
"open",
|
|
59
|
+
"--limit",
|
|
60
|
+
"200",
|
|
61
|
+
"--json",
|
|
62
|
+
"number,title",
|
|
63
|
+
]);
|
|
64
|
+
const issues = JSON.parse(output);
|
|
65
|
+
return issues.find((issue) => issue.title === ISSUE_TITLE)?.number ?? null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (!existsSync(auditFilePath)) {
|
|
69
|
+
console.log(`No audit file found at ${auditFilePath}; skipping issue update.`);
|
|
70
|
+
process.exit(0);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const summary = readAuditSummary(auditFilePath);
|
|
74
|
+
if (summary.total === 0) {
|
|
75
|
+
console.log("No vulnerabilities found; skipping issue create/update.");
|
|
76
|
+
process.exit(0);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const body = buildBody(summary);
|
|
80
|
+
const tempFile = path.join(os.tmpdir(), `npm-audit-issue-${Date.now()}.md`);
|
|
81
|
+
writeFileSync(tempFile, body, "utf8");
|
|
82
|
+
|
|
83
|
+
const existingIssueNumber = findExistingIssueNumber();
|
|
84
|
+
if (existingIssueNumber) {
|
|
85
|
+
runGh(["issue", "comment", String(existingIssueNumber), "--body-file", tempFile]);
|
|
86
|
+
console.log(`Updated existing issue #${existingIssueNumber}.`);
|
|
87
|
+
} else {
|
|
88
|
+
runGh(["issue", "create", "--title", ISSUE_TITLE, "--body-file", tempFile]);
|
|
89
|
+
console.log("Created new npm audit tracking issue.");
|
|
90
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { createBadgeSvg, statusToColor, statusToText } from "../lib/svg.mjs";
|
|
4
|
+
|
|
5
|
+
const repo = process.env.GITHUB_REPOSITORY;
|
|
6
|
+
const token = process.env.GITHUB_TOKEN;
|
|
7
|
+
const outputDir =
|
|
8
|
+
process.env.CI_BADGE_OUTPUT_DIR ?? path.join("projects", "ci-tools", "dist", "ci", "badges");
|
|
9
|
+
|
|
10
|
+
if (repo == null || token == null) {
|
|
11
|
+
throw new Error("GITHUB_REPOSITORY and GITHUB_TOKEN are required");
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const workflowFileName = "ci.yml";
|
|
15
|
+
|
|
16
|
+
const branches = [
|
|
17
|
+
{ branch: "main", label: "CI Main", file: "ci-main.svg" },
|
|
18
|
+
{ branch: "release", label: "CI Release", file: "ci-release.svg" },
|
|
19
|
+
];
|
|
20
|
+
|
|
21
|
+
async function getLatestWorkflowRun(branch) {
|
|
22
|
+
const url =
|
|
23
|
+
`https://api.github.com/repos/${repo}/actions/workflows/${workflowFileName}/runs` +
|
|
24
|
+
`?branch=${branch}&event=push&status=completed&per_page=1`;
|
|
25
|
+
const response = await fetch(url, {
|
|
26
|
+
headers: {
|
|
27
|
+
Authorization: `Bearer ${token}`,
|
|
28
|
+
Accept: "application/vnd.github+json",
|
|
29
|
+
},
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
if (!response.ok) {
|
|
33
|
+
process.stderr.write(
|
|
34
|
+
`Warning: Failed to fetch workflow runs for ${branch}: ${response.status}. Using unknown badge status.\n`,
|
|
35
|
+
);
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const data = await response.json();
|
|
40
|
+
return data.workflow_runs?.[0] ?? null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
await mkdir(outputDir, { recursive: true });
|
|
44
|
+
|
|
45
|
+
for (const item of branches) {
|
|
46
|
+
const latestRun = await getLatestWorkflowRun(item.branch);
|
|
47
|
+
const status = latestRun?.status ?? "completed";
|
|
48
|
+
const conclusion = latestRun?.conclusion ?? "unknown";
|
|
49
|
+
const value = statusToText(status, conclusion);
|
|
50
|
+
const color = statusToColor(status, conclusion);
|
|
51
|
+
await writeFile(path.join(outputDir, item.file), createBadgeSvg(item.label, value, color));
|
|
52
|
+
}
|
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* bench.mjs — run the forge benchmark and compare against the last saved baseline.
|
|
5
|
+
*
|
|
6
|
+
* Usage:
|
|
7
|
+
* node bench.mjs — run and compare, do not save
|
|
8
|
+
* node bench.mjs save — run, compare, and save as the new baseline
|
|
9
|
+
* node bench.mjs report — run and compare, do not save
|
|
10
|
+
* BENCH_ITERATIONS=3 node bench.mjs
|
|
11
|
+
*
|
|
12
|
+
* Context ID
|
|
13
|
+
* ----------
|
|
14
|
+
* 16-char SHA-256 prefix of: platform|arch|cpu|cores|memGB|node
|
|
15
|
+
* Ensures history is always compared like-for-like across machines.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { spawnSync } from "node:child_process";
|
|
19
|
+
import crypto from "node:crypto";
|
|
20
|
+
import {
|
|
21
|
+
existsSync,
|
|
22
|
+
mkdirSync,
|
|
23
|
+
mkdtempSync,
|
|
24
|
+
readFileSync,
|
|
25
|
+
rmSync,
|
|
26
|
+
writeFileSync,
|
|
27
|
+
} from "node:fs";
|
|
28
|
+
import os from "node:os";
|
|
29
|
+
import { join } from "node:path";
|
|
30
|
+
|
|
31
|
+
const WORKSPACE_ROOT = process.cwd();
|
|
32
|
+
const HISTORY_FILE = join(
|
|
33
|
+
WORKSPACE_ROOT,
|
|
34
|
+
"projects/ci-tools/dist/ci/bench/bench-history.json",
|
|
35
|
+
);
|
|
36
|
+
const VITEST_BIN = join(WORKSPACE_ROOT, "node_modules/.bin/vitest");
|
|
37
|
+
const BENCH_FILE = "projects/terminal/e2e/build.benchmark.bench.ts";
|
|
38
|
+
const CONFIG = join(WORKSPACE_ROOT, "vitest.e2e.config.ts");
|
|
39
|
+
const TRACE_LOG_DIR = join(WORKSPACE_ROOT, ".decohere", "logs");
|
|
40
|
+
const MAX_RUNS = 10;
|
|
41
|
+
const command = process.argv[2] ?? "run";
|
|
42
|
+
|
|
43
|
+
// ── Context ───────────────────────────────────────────────────────────────────
|
|
44
|
+
|
|
45
|
+
function getContext() {
|
|
46
|
+
const cpus = os.cpus();
|
|
47
|
+
const nodeMajor = process.version.match(/^v(\d+)/)?.[1] ?? "?";
|
|
48
|
+
const rawEnv = process.env.BENCH_ENV ?? null;
|
|
49
|
+
return {
|
|
50
|
+
benchEnv: rawEnv ? `${rawEnv}-node${nodeMajor}` : "(local)",
|
|
51
|
+
date: new Date().toISOString(),
|
|
52
|
+
os: { platform: os.platform(), arch: os.arch() },
|
|
53
|
+
cpu: { model: cpus[0]?.model?.trim() ?? "unknown", count: cpus.length },
|
|
54
|
+
memoryGb: parseFloat((os.totalmem() / 1024 ** 3).toFixed(1)),
|
|
55
|
+
node: process.version,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function contextId(ctx) {
|
|
60
|
+
const key = [
|
|
61
|
+
ctx.os.platform,
|
|
62
|
+
ctx.os.arch,
|
|
63
|
+
ctx.cpu.model,
|
|
64
|
+
ctx.cpu.count,
|
|
65
|
+
ctx.memoryGb,
|
|
66
|
+
ctx.node,
|
|
67
|
+
].join("|");
|
|
68
|
+
return crypto.createHash("sha256").update(key).digest("hex").slice(0, 16);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function truthy(value, defaultValue = false) {
|
|
72
|
+
if (value == null) return defaultValue;
|
|
73
|
+
return !["0", "false", "no", "off"].includes(String(value).toLowerCase());
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function toFileTimestamp(date = new Date()) {
|
|
77
|
+
const pad = (n) => String(n).padStart(2, "0");
|
|
78
|
+
return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function toTraceSample(line, parsed, durationMs) {
|
|
82
|
+
return {
|
|
83
|
+
time: parsed.time,
|
|
84
|
+
name: parsed.name,
|
|
85
|
+
message: parsed.msg,
|
|
86
|
+
durationMs,
|
|
87
|
+
line,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function parseTraceSamples(rawOutput) {
|
|
92
|
+
const samples = [];
|
|
93
|
+
for (const rawLine of rawOutput.split(/\r?\n/)) {
|
|
94
|
+
const line = rawLine.trim();
|
|
95
|
+
if (!line.startsWith("{")) continue;
|
|
96
|
+
|
|
97
|
+
let parsed;
|
|
98
|
+
try {
|
|
99
|
+
parsed = JSON.parse(line);
|
|
100
|
+
} catch {
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if (typeof parsed?.name !== "string" || typeof parsed?.msg !== "string") continue;
|
|
105
|
+
const match = parsed.msg.match(/^(.*):\s+([0-9]+(?:\.[0-9]+)?)ms$/);
|
|
106
|
+
if (match == null) continue;
|
|
107
|
+
|
|
108
|
+
const durationMs = Number(match[2]);
|
|
109
|
+
if (!Number.isFinite(durationMs)) continue;
|
|
110
|
+
|
|
111
|
+
samples.push({
|
|
112
|
+
className: parsed.name,
|
|
113
|
+
methodLabel: match[1],
|
|
114
|
+
methodName: match[1].split("(")[0],
|
|
115
|
+
durationMs,
|
|
116
|
+
sample: toTraceSample(rawLine, parsed, durationMs),
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
return samples;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function aggregateSamples(samples, keyBuilder) {
|
|
123
|
+
const aggregates = new Map();
|
|
124
|
+
for (const sample of samples) {
|
|
125
|
+
const key = keyBuilder(sample);
|
|
126
|
+
const current = aggregates.get(key) ?? {
|
|
127
|
+
key,
|
|
128
|
+
hitCount: 0,
|
|
129
|
+
totalMs: 0,
|
|
130
|
+
maxMs: 0,
|
|
131
|
+
};
|
|
132
|
+
current.hitCount += 1;
|
|
133
|
+
current.totalMs += sample.durationMs;
|
|
134
|
+
if (sample.durationMs > current.maxMs) current.maxMs = sample.durationMs;
|
|
135
|
+
aggregates.set(key, current);
|
|
136
|
+
}
|
|
137
|
+
return [...aggregates.values()];
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function analyzeTraceOutput(rawOutput) {
|
|
141
|
+
const samples = parseTraceSamples(rawOutput);
|
|
142
|
+
if (samples.length === 0) {
|
|
143
|
+
process.stderr.write("\n⚠ No structured trace samples found to analyze.\n");
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const byMethod = aggregateSamples(
|
|
148
|
+
samples,
|
|
149
|
+
(sample) => `${sample.className}.${sample.methodName}`,
|
|
150
|
+
);
|
|
151
|
+
const byClass = aggregateSamples(samples, (sample) => sample.className);
|
|
152
|
+
const byHits = [...byMethod].sort((a, b) => b.hitCount - a.hitCount).slice(0, 10);
|
|
153
|
+
const byTotal = [...byMethod].sort((a, b) => b.totalMs - a.totalMs).slice(0, 10);
|
|
154
|
+
const byMax = [...byMethod].sort((a, b) => b.maxMs - a.maxMs).slice(0, 10);
|
|
155
|
+
const byClassTotal = [...byClass].sort((a, b) => b.totalMs - a.totalMs).slice(0, 5);
|
|
156
|
+
const totalTraceMs = samples.reduce((sum, sample) => sum + sample.durationMs, 0);
|
|
157
|
+
const hottest = [...samples].sort((a, b) => b.durationMs - a.durationMs).slice(0, 5);
|
|
158
|
+
|
|
159
|
+
process.stderr.write(
|
|
160
|
+
"\n┌─ Trace Analysis ─────────────────────────────────────────────────────────\n",
|
|
161
|
+
);
|
|
162
|
+
process.stderr.write(`│ Samples : ${samples.length}\n`);
|
|
163
|
+
process.stderr.write(`│ Unique funcs : ${byMethod.length}\n`);
|
|
164
|
+
process.stderr.write(`│ Total traced : ${totalTraceMs.toFixed(2)}ms\n`);
|
|
165
|
+
process.stderr.write(`│ Avg sample : ${(totalTraceMs / samples.length).toFixed(3)}ms\n`);
|
|
166
|
+
process.stderr.write(
|
|
167
|
+
"└──────────────────────────────────────────────────────────────────────────\n",
|
|
168
|
+
);
|
|
169
|
+
|
|
170
|
+
const printTable = (title, rows, valueFormatter) => {
|
|
171
|
+
process.stderr.write(`\n${title}\n`);
|
|
172
|
+
for (const row of rows) {
|
|
173
|
+
process.stderr.write(
|
|
174
|
+
` ${valueFormatter(row).padStart(10)} ${row.key} (hits=${row.hitCount}, max=${row.maxMs.toFixed(2)}ms)\n`,
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
printTable("Top Functions By Hit Count", byHits, (row) => String(row.hitCount));
|
|
180
|
+
printTable("Top Functions By Total Time", byTotal, (row) => `${row.totalMs.toFixed(2)}ms`);
|
|
181
|
+
printTable(
|
|
182
|
+
"Top Functions By Worst Single Call",
|
|
183
|
+
byMax,
|
|
184
|
+
(row) => `${row.maxMs.toFixed(2)}ms`,
|
|
185
|
+
);
|
|
186
|
+
printTable(
|
|
187
|
+
"Top Classes By Total Trace Time",
|
|
188
|
+
byClassTotal,
|
|
189
|
+
(row) => `${row.totalMs.toFixed(2)}ms`,
|
|
190
|
+
);
|
|
191
|
+
|
|
192
|
+
process.stderr.write("\nSlowest Individual Samples\n");
|
|
193
|
+
for (const hot of hottest) {
|
|
194
|
+
process.stderr.write(
|
|
195
|
+
` ${hot.durationMs.toFixed(2).padStart(10)}ms ${hot.className}.${hot.methodLabel}\n`,
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function getBenchmarks(vitestOutput) {
|
|
201
|
+
return vitestOutput.files
|
|
202
|
+
.flatMap((file) => file.groups)
|
|
203
|
+
.flatMap((group) => group.benchmarks);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function printBenchmarkComparison(currentRun, previousRun) {
|
|
207
|
+
if (previousRun == null) {
|
|
208
|
+
process.stderr.write("No previous saved run for this context; skipping comparison.\n");
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const currentBenchmarks = getBenchmarks(currentRun.vitestOutput);
|
|
213
|
+
const previousByName = new Map(
|
|
214
|
+
getBenchmarks(previousRun.vitestOutput).map((benchmark) => [benchmark.name, benchmark]),
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
process.stderr.write("Current Run vs Previous\n");
|
|
218
|
+
process.stderr.write(` current : ${currentRun.date}\n`);
|
|
219
|
+
process.stderr.write(` previous: ${previousRun.date}\n`);
|
|
220
|
+
process.stderr.write("\n");
|
|
221
|
+
|
|
222
|
+
const rows = currentBenchmarks.map((benchmark) => {
|
|
223
|
+
const previous = previousByName.get(benchmark.name);
|
|
224
|
+
if (previous == null) {
|
|
225
|
+
return {
|
|
226
|
+
name: benchmark.name,
|
|
227
|
+
current: `${benchmark.mean.toFixed(2)}ms`,
|
|
228
|
+
previous: "n/a",
|
|
229
|
+
delta: "n/a",
|
|
230
|
+
change: "n/a",
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const delta = benchmark.mean - previous.mean;
|
|
235
|
+
const pct = previous.mean === 0 ? 0 : (delta / previous.mean) * 100;
|
|
236
|
+
const deltaSign = delta >= 0 ? "+" : "";
|
|
237
|
+
return {
|
|
238
|
+
name: benchmark.name,
|
|
239
|
+
current: `${benchmark.mean.toFixed(2)}ms`,
|
|
240
|
+
previous: `${previous.mean.toFixed(2)}ms`,
|
|
241
|
+
delta: `${deltaSign}${delta.toFixed(2)}ms`,
|
|
242
|
+
change: `${deltaSign}${pct.toFixed(2)}%`,
|
|
243
|
+
};
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
const headers = {
|
|
247
|
+
name: "Benchmark",
|
|
248
|
+
current: "Current",
|
|
249
|
+
previous: "Previous",
|
|
250
|
+
delta: "Delta",
|
|
251
|
+
change: "%",
|
|
252
|
+
};
|
|
253
|
+
|
|
254
|
+
const widths = {
|
|
255
|
+
name: Math.max(headers.name.length, ...rows.map((row) => row.name.length)),
|
|
256
|
+
current: Math.max(headers.current.length, ...rows.map((row) => row.current.length)),
|
|
257
|
+
previous: Math.max(headers.previous.length, ...rows.map((row) => row.previous.length)),
|
|
258
|
+
delta: Math.max(headers.delta.length, ...rows.map((row) => row.delta.length)),
|
|
259
|
+
change: Math.max(headers.change.length, ...rows.map((row) => row.change.length)),
|
|
260
|
+
};
|
|
261
|
+
|
|
262
|
+
const divider = [
|
|
263
|
+
"-".repeat(widths.name),
|
|
264
|
+
"-".repeat(widths.current),
|
|
265
|
+
"-".repeat(widths.previous),
|
|
266
|
+
"-".repeat(widths.delta),
|
|
267
|
+
"-".repeat(widths.change),
|
|
268
|
+
].join("-+-");
|
|
269
|
+
|
|
270
|
+
const formatRow = (row) =>
|
|
271
|
+
[
|
|
272
|
+
row.name.padEnd(widths.name),
|
|
273
|
+
row.current.padStart(widths.current),
|
|
274
|
+
row.previous.padStart(widths.previous),
|
|
275
|
+
row.delta.padStart(widths.delta),
|
|
276
|
+
row.change.padStart(widths.change),
|
|
277
|
+
].join(" | ");
|
|
278
|
+
|
|
279
|
+
process.stderr.write(`${formatRow(headers)}\n`);
|
|
280
|
+
process.stderr.write(`${divider}\n`);
|
|
281
|
+
for (const row of rows) {
|
|
282
|
+
process.stderr.write(`${formatRow(row)}\n`);
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
process.stderr.write("\n");
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// ── Run ───────────────────────────────────────────────────────────────────────
|
|
289
|
+
|
|
290
|
+
const ctx = getContext();
|
|
291
|
+
const id = contextId(ctx);
|
|
292
|
+
const iterations = process.env.BENCH_ITERATIONS ?? "5";
|
|
293
|
+
const traceEnabled = process.env.FORGE_TRACE != null;
|
|
294
|
+
const shouldSave = truthy(process.env.BENCH_SAVE, command === "save");
|
|
295
|
+
const shouldAnalyzeTrace = traceEnabled && truthy(process.env.BENCH_ANALYZE_TRACE, true);
|
|
296
|
+
const shouldCollectTraceLog = traceEnabled && truthy(process.env.BENCH_TRACE_LOG, true);
|
|
297
|
+
const traceLogPath = shouldCollectTraceLog
|
|
298
|
+
? join(TRACE_LOG_DIR, `perf-trace-${toFileTimestamp()}.log`)
|
|
299
|
+
: null;
|
|
300
|
+
|
|
301
|
+
if (traceLogPath != null) {
|
|
302
|
+
mkdirSync(TRACE_LOG_DIR, { recursive: true });
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
process.stderr.write(
|
|
306
|
+
[
|
|
307
|
+
"\n┌─ Benchmark Environment ─────────────────────────────────────────────────",
|
|
308
|
+
`│ Context ID : ${id}`,
|
|
309
|
+
`│ BENCH_ENV : ${ctx.benchEnv}`,
|
|
310
|
+
`│ Date : ${ctx.date}`,
|
|
311
|
+
`│ OS : ${ctx.os.platform} ${ctx.os.arch}`,
|
|
312
|
+
`│ CPU : ${ctx.cpu.model} × ${ctx.cpu.count}`,
|
|
313
|
+
`│ Memory : ${ctx.memoryGb} GB`,
|
|
314
|
+
`│ Node : ${ctx.node}`,
|
|
315
|
+
"└──────────────────────────────────────────────────────────────────────────\n\n",
|
|
316
|
+
].join("\n"),
|
|
317
|
+
);
|
|
318
|
+
|
|
319
|
+
if (traceEnabled) {
|
|
320
|
+
process.stderr.write("Trace mode enabled with structured pino logs for analysis.\n\n");
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
const tmpDir = mkdtempSync(`${os.tmpdir()}/ark-bench-`);
|
|
324
|
+
const outJson = join(tmpDir, "output.json");
|
|
325
|
+
|
|
326
|
+
try {
|
|
327
|
+
const result = spawnSync(
|
|
328
|
+
VITEST_BIN,
|
|
329
|
+
["bench", "--config", CONFIG, "--run", "--outputJson", outJson, BENCH_FILE],
|
|
330
|
+
{
|
|
331
|
+
cwd: WORKSPACE_ROOT,
|
|
332
|
+
stdio: traceEnabled ? "pipe" : "inherit",
|
|
333
|
+
encoding: traceEnabled ? "utf8" : undefined,
|
|
334
|
+
maxBuffer: 1024 * 1024 * 200,
|
|
335
|
+
env: {
|
|
336
|
+
...process.env,
|
|
337
|
+
NODE_OPTIONS: "",
|
|
338
|
+
BENCH_ITERATIONS: iterations,
|
|
339
|
+
...(traceEnabled
|
|
340
|
+
? { FORGE_TRACE_STRUCTURED: "1", LOG_LEVEL: process.env.LOG_LEVEL ?? "trace" }
|
|
341
|
+
: {}),
|
|
342
|
+
},
|
|
343
|
+
},
|
|
344
|
+
);
|
|
345
|
+
|
|
346
|
+
if (traceEnabled) {
|
|
347
|
+
const stdout = typeof result.stdout === "string" ? result.stdout : "";
|
|
348
|
+
const stderr = typeof result.stderr === "string" ? result.stderr : "";
|
|
349
|
+
if (stdout.length > 0) process.stdout.write(stdout);
|
|
350
|
+
if (stderr.length > 0) process.stderr.write(stderr);
|
|
351
|
+
|
|
352
|
+
const combined = `${stdout}${stderr.length > 0 ? `\n${stderr}` : ""}`;
|
|
353
|
+
if (traceLogPath != null) {
|
|
354
|
+
writeFileSync(traceLogPath, combined);
|
|
355
|
+
process.stderr.write(`\n✓ Trace log written: ${traceLogPath}\n`);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
if (shouldAnalyzeTrace) {
|
|
359
|
+
analyzeTraceOutput(combined);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
if (result.status !== 0) process.exit(result.status ?? 1);
|
|
364
|
+
|
|
365
|
+
if (!existsSync(outJson)) {
|
|
366
|
+
process.stderr.write("✗ No output JSON — bench may have errored internally.\n");
|
|
367
|
+
process.exit(1);
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
const output = JSON.parse(readFileSync(outJson, "utf8"));
|
|
371
|
+
const benches = output.files.flatMap((f) => f.groups).flatMap((g) => g.benchmarks);
|
|
372
|
+
const currentRun = { date: ctx.date, context: ctx, vitestOutput: output };
|
|
373
|
+
|
|
374
|
+
// Fail loud on NaN — vitest swallows errors in bench callbacks silently
|
|
375
|
+
const invalid = benches.filter((b) => !Number.isFinite(b.mean) || !b.sampleCount);
|
|
376
|
+
if (invalid.length > 0) {
|
|
377
|
+
process.stderr.write(
|
|
378
|
+
`✗ ${invalid.length} benchmark(s) have NaN results — check for errors in the bench callback:\n` +
|
|
379
|
+
invalid.map((b) => ` • ${b.name}`).join("\n") +
|
|
380
|
+
"\n",
|
|
381
|
+
);
|
|
382
|
+
process.exit(1);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
// ── Save ──────────────────────────────────────────────────────────────────
|
|
386
|
+
|
|
387
|
+
const history = existsSync(HISTORY_FILE)
|
|
388
|
+
? JSON.parse(readFileSync(HISTORY_FILE, "utf8"))
|
|
389
|
+
: { runs: {} };
|
|
390
|
+
|
|
391
|
+
if (!history.runs[id]) history.runs[id] = { context: ctx, runs: [] };
|
|
392
|
+
const previousRun = history.runs[id].runs[0];
|
|
393
|
+
|
|
394
|
+
printBenchmarkComparison(currentRun, previousRun);
|
|
395
|
+
|
|
396
|
+
if (!shouldSave) {
|
|
397
|
+
process.stderr.write(`✓ Not saved (mode: ${command}, baseline unchanged)\n\n`);
|
|
398
|
+
process.exit(0);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
history.runs[id].context = ctx;
|
|
402
|
+
history.runs[id].runs.unshift(currentRun);
|
|
403
|
+
if (history.runs[id].runs.length > MAX_RUNS) history.runs[id].runs.length = MAX_RUNS;
|
|
404
|
+
|
|
405
|
+
writeFileSync(HISTORY_FILE, JSON.stringify(history, null, 2));
|
|
406
|
+
|
|
407
|
+
const stored = history.runs[id].runs.length;
|
|
408
|
+
process.stderr.write(`\n✓ Saved (context: ${id}, runs stored: ${stored}/${MAX_RUNS})\n\n`);
|
|
409
|
+
} finally {
|
|
410
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
411
|
+
}
|