@davesheffer/hunch 1.8.2 → 1.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -1
- package/dist/cli/index.js +1238 -396
- package/dist/constitution/adapters.js +31 -14
- package/dist/constitution/behaviorEvaluator.js +20 -7
- package/dist/constitution/behaviorProof.js +3 -2
- package/dist/constitution/canonical.js +7 -1
- package/dist/constitution/card.js +7 -2
- package/dist/constitution/compiler.js +71 -1
- package/dist/constitution/correctionPolicyMaterializer.js +496 -0
- package/dist/constitution/delta.js +3 -2
- package/dist/constitution/evaluator.js +29 -3
- package/dist/constitution/experiment.js +96 -5
- package/dist/constitution/experimentRunner.js +43 -14
- package/dist/constitution/g2BehaviorCandidates.js +49 -26
- package/dist/constitution/g2BehaviorDependencies.js +203 -14
- package/dist/constitution/g2Candidates.js +1 -1
- package/dist/constitution/lifecycle.js +17 -0
- package/dist/constitution/plan.js +26 -9
- package/dist/constitution/replacementFreeGit.js +67 -0
- package/dist/constitution/replay.js +6 -0
- package/dist/constitution/replayCache.js +1 -1
- package/dist/constitution/replayWorker.js +1 -1
- package/dist/constitution/repository.js +141 -5
- package/dist/constitution/safeCheckout.js +75 -0
- package/dist/constitution/schema.js +30 -5
- package/dist/constitution/service.js +74 -14
- package/dist/constitution/sourceMutation.js +65 -12
- package/dist/constitution/staticGraphBaseline.js +44 -0
- package/dist/constitution/structural.js +60 -4
- package/dist/core/autoreview.js +1 -1
- package/dist/core/canonicalOrder.js +6 -0
- package/dist/core/conformance.js +68 -27
- package/dist/core/docscan.js +2 -1
- package/dist/core/escalations.js +11 -0
- package/dist/core/io.js +44 -9
- package/dist/core/overlaySafety.js +178 -0
- package/dist/core/paths.js +13 -2
- package/dist/core/safeRepoFile.js +74 -0
- package/dist/extractors/comments.js +6 -8
- package/dist/extractors/git.js +1631 -82
- package/dist/extractors/indexer.js +86 -47
- package/dist/extractors/repoSource.js +390 -0
- package/dist/integrations/ciAction.js +10 -2
- package/dist/integrations/gitignore.js +44 -5
- package/dist/integrations/mergeDriver.js +23 -5
- package/dist/integrations/sync.js +61 -5
- package/dist/integrations/team.js +666 -23
- package/dist/mcp/server.js +261 -34
- package/dist/store/db.js +57 -7
- package/dist/store/hunchStore.js +92 -11
- package/dist/store/jsonStore.js +350 -63
- package/dist/store/schema.js +27 -11
- package/dist/synthesis/provider.js +13 -4
- package/dist/synthesis/synthesize.js +56 -19
- package/dist/wiki/graph.js +5 -4
- package/dist/wiki/wiki.js +16 -10
- package/package.json +15 -3
- package/tooling/competitive-watch.mjs +108 -0
- package/tooling/md1-benchmark.mjs +628 -0
|
@@ -0,0 +1,628 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { spawnSync } from "node:child_process";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
import {
|
|
5
|
+
existsSync,
|
|
6
|
+
mkdirSync,
|
|
7
|
+
mkdtempSync,
|
|
8
|
+
readFileSync,
|
|
9
|
+
readdirSync,
|
|
10
|
+
realpathSync,
|
|
11
|
+
renameSync,
|
|
12
|
+
rmSync,
|
|
13
|
+
symlinkSync,
|
|
14
|
+
writeFileSync,
|
|
15
|
+
} from "node:fs";
|
|
16
|
+
import { cpus, tmpdir, totalmem } from "node:os";
|
|
17
|
+
import { dirname, join, resolve } from "node:path";
|
|
18
|
+
import process from "node:process";
|
|
19
|
+
import { fileURLToPath } from "node:url";
|
|
20
|
+
|
|
21
|
+
export const BENCHMARK_BASELINE_REF = "610cd2e5673bf3c69ac984b4737e2d4a749ed374";
|
|
22
|
+
export const BENCHMARK_SCHEMA = "hunch.md1a.performance.v1";
|
|
23
|
+
|
|
24
|
+
const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
25
|
+
const tsxCli = join(projectRoot, "node_modules", "tsx", "dist", "cli.mjs");
|
|
26
|
+
const FIXTURE_DATE = "2026-07-10T10:00:00.000Z";
|
|
27
|
+
const FIXTURE_GIT_DATE = "2026-07-10T10:00:00Z";
|
|
28
|
+
const DEFAULTS = Object.freeze({ samples: 10, many: 16, files: 48 });
|
|
29
|
+
|
|
30
|
+
function sha256(value) {
|
|
31
|
+
return createHash("sha256").update(value).digest("hex");
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function stable(value) {
|
|
35
|
+
if (Array.isArray(value)) return value.map(stable);
|
|
36
|
+
if (value && typeof value === "object") {
|
|
37
|
+
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stable(value[key])]));
|
|
38
|
+
}
|
|
39
|
+
return value;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function integer(value, flag, minimum) {
|
|
43
|
+
if (!/^\d+$/.test(value ?? "")) throw new Error(`${flag} requires an integer`);
|
|
44
|
+
const parsed = Number(value);
|
|
45
|
+
if (!Number.isSafeInteger(parsed) || parsed < minimum) throw new Error(`${flag} must be at least ${minimum}`);
|
|
46
|
+
return parsed;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function help() {
|
|
50
|
+
return [
|
|
51
|
+
"Deterministic MD-1a baseline/current performance benchmark.",
|
|
52
|
+
"",
|
|
53
|
+
"Usage: npm run bench:md1 -- [options]",
|
|
54
|
+
"",
|
|
55
|
+
` --samples <n> timed samples per case (default ${DEFAULTS.samples})`,
|
|
56
|
+
` --many <n> bounded-many correction count (default ${DEFAULTS.many})`,
|
|
57
|
+
` --files <n> deterministic TypeScript fixture files (default ${DEFAULTS.files})`,
|
|
58
|
+
" --case <selector> run only operation:home:count; repeatable",
|
|
59
|
+
" operations: index, sync; homes: public, split_private",
|
|
60
|
+
" count: 0, 1, many, or the exact --many value",
|
|
61
|
+
" --output <file> atomically write the JSON receipt in addition to stdout",
|
|
62
|
+
" --keep-temp retain disposable fixtures (path is printed to stderr)",
|
|
63
|
+
" --help show this help",
|
|
64
|
+
"",
|
|
65
|
+
"The benchmark uses no remotes, forces deterministic synthesis, and routes all",
|
|
66
|
+
"proxy variables to a closed loopback port. Fixture construction and one warm-up",
|
|
67
|
+
"are outside timed samples. Peak RSS uses /usr/bin/time on macOS or Linux.",
|
|
68
|
+
].join("\n");
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function parseBenchmarkArgs(argv) {
|
|
72
|
+
const parsed = { ...DEFAULTS, cases: [], output: null, keepTemp: false, help: false };
|
|
73
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
74
|
+
const arg = argv[index];
|
|
75
|
+
if (arg === "--help" || arg === "-h") parsed.help = true;
|
|
76
|
+
else if (arg === "--keep-temp") parsed.keepTemp = true;
|
|
77
|
+
else if (["--samples", "--many", "--files", "--case", "--output"].includes(arg)) {
|
|
78
|
+
const value = argv[index + 1];
|
|
79
|
+
if (!value) throw new Error(`${arg} requires a value`);
|
|
80
|
+
if (arg === "--samples") parsed.samples = integer(value, arg, 1);
|
|
81
|
+
else if (arg === "--many") parsed.many = integer(value, arg, 2);
|
|
82
|
+
else if (arg === "--files") parsed.files = integer(value, arg, 2);
|
|
83
|
+
else if (arg === "--case") parsed.cases.push(value);
|
|
84
|
+
else parsed.output = value;
|
|
85
|
+
index += 1;
|
|
86
|
+
} else throw new Error(`unknown benchmark argument: ${arg}`);
|
|
87
|
+
}
|
|
88
|
+
if (parsed.files < parsed.many) throw new Error("--files must be at least --many so every correction has a distinct concrete file");
|
|
89
|
+
return parsed;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function benchmarkScenarios(many) {
|
|
93
|
+
const scenarios = [];
|
|
94
|
+
for (const operation of ["index", "sync"]) {
|
|
95
|
+
for (const home of ["public", "split_private"]) {
|
|
96
|
+
for (const activeCorrections of [0, 1, many]) {
|
|
97
|
+
scenarios.push({
|
|
98
|
+
id: `${operation}.${home}.${activeCorrections}`,
|
|
99
|
+
operation,
|
|
100
|
+
home,
|
|
101
|
+
active_corrections: activeCorrections,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return scenarios;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function selectScenarios(options) {
|
|
110
|
+
const all = benchmarkScenarios(options.many);
|
|
111
|
+
if (!options.cases.length) return all;
|
|
112
|
+
const selected = new Set(options.cases.map((selector) => {
|
|
113
|
+
const [operation, home, rawCount, extra] = selector.split(":");
|
|
114
|
+
const count = rawCount === "many" ? options.many : Number(rawCount);
|
|
115
|
+
if (extra !== undefined || !["index", "sync"].includes(operation)
|
|
116
|
+
|| !["public", "split_private"].includes(home)
|
|
117
|
+
|| ![0, 1, options.many].includes(count)) {
|
|
118
|
+
throw new Error(`invalid --case ${selector}; expected operation:home:count from the configured matrix`);
|
|
119
|
+
}
|
|
120
|
+
return `${operation}.${home}.${count}`;
|
|
121
|
+
}));
|
|
122
|
+
return all.filter((scenario) => selected.has(scenario.id));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function nearestRank(values, percentile) {
|
|
126
|
+
if (!values.length) throw new Error("cannot summarize an empty sample set");
|
|
127
|
+
const ordered = [...values].sort((left, right) => left - right);
|
|
128
|
+
return ordered[Math.max(0, Math.ceil(percentile * ordered.length) - 1)];
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export function summarizeSamples(samples) {
|
|
132
|
+
if (!samples.length) throw new Error("cannot summarize an empty sample set");
|
|
133
|
+
const summary = (key) => {
|
|
134
|
+
const values = samples.map((sample) => sample[key]);
|
|
135
|
+
return {
|
|
136
|
+
min: Math.min(...values),
|
|
137
|
+
p50: nearestRank(values, 0.5),
|
|
138
|
+
p95: nearestRank(values, 0.95),
|
|
139
|
+
max: Math.max(...values),
|
|
140
|
+
};
|
|
141
|
+
};
|
|
142
|
+
return { wall_ms: summary("wall_ms"), peak_rss_bytes: summary("peak_rss_bytes") };
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function run(command, args, options = {}) {
|
|
146
|
+
const child = spawnSync(command, args, {
|
|
147
|
+
encoding: "utf8",
|
|
148
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
149
|
+
timeout: 300_000,
|
|
150
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
151
|
+
...options,
|
|
152
|
+
});
|
|
153
|
+
if (child.error) throw child.error;
|
|
154
|
+
if (child.status !== 0) {
|
|
155
|
+
const detail = [child.stderr, child.stdout].map((value) => value?.trim()).filter(Boolean).join("\n");
|
|
156
|
+
throw new Error(`${command} ${args.join(" ")} failed with exit ${child.status}${detail ? `:\n${detail}` : ""}`);
|
|
157
|
+
}
|
|
158
|
+
return { stdout: child.stdout, stderr: child.stderr, status: child.status };
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function git(cwd, args, env = {}) {
|
|
162
|
+
return run("git", args, { cwd, env: { ...process.env, ...env } }).stdout.trim();
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function write(root, file, value) {
|
|
166
|
+
const target = join(root, file);
|
|
167
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
168
|
+
writeFileSync(target, value);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function writeHunchJson(file, value) {
|
|
172
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
173
|
+
const temporary = `${file}.tmp-${process.pid}`;
|
|
174
|
+
writeFileSync(temporary, JSON.stringify(value, null, 2) + "\n");
|
|
175
|
+
renameSync(temporary, file);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function walkFiles(root, relativeRoot = "") {
|
|
179
|
+
const dir = join(root, relativeRoot);
|
|
180
|
+
if (!existsSync(dir)) return [];
|
|
181
|
+
const files = [];
|
|
182
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
183
|
+
const rel = join(relativeRoot, entry.name);
|
|
184
|
+
if (entry.isDirectory()) files.push(...walkFiles(root, rel));
|
|
185
|
+
else if (entry.isFile()) files.push(rel);
|
|
186
|
+
}
|
|
187
|
+
return files.sort();
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function sourceHash(root) {
|
|
191
|
+
const files = [
|
|
192
|
+
...walkFiles(root, "src"),
|
|
193
|
+
...["package.json", "package-lock.json"].filter((file) => existsSync(join(root, file))),
|
|
194
|
+
].sort();
|
|
195
|
+
const hash = createHash("sha256");
|
|
196
|
+
for (const file of files) {
|
|
197
|
+
hash.update(file.replaceAll("\\", "/"));
|
|
198
|
+
hash.update("\0");
|
|
199
|
+
hash.update(readFileSync(join(root, file)));
|
|
200
|
+
hash.update("\0");
|
|
201
|
+
}
|
|
202
|
+
return `sha256:${hash.digest("hex")}`;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function baselineCheckout(temp) {
|
|
206
|
+
const commit = git(projectRoot, ["rev-parse", `${BENCHMARK_BASELINE_REF}^{commit}`]);
|
|
207
|
+
if (commit !== BENCHMARK_BASELINE_REF) throw new Error(`pinned baseline ${BENCHMARK_BASELINE_REF} did not resolve exactly`);
|
|
208
|
+
const target = join(temp, "baseline-source");
|
|
209
|
+
run("git", ["clone", "-q", "--shared", "--no-checkout", projectRoot, target]);
|
|
210
|
+
git(target, ["checkout", "-q", "--detach", BENCHMARK_BASELINE_REF]);
|
|
211
|
+
if (!existsSync(join(projectRoot, "node_modules"))) throw new Error("node_modules is missing; run npm install before benchmarking");
|
|
212
|
+
symlinkSync(join(projectRoot, "node_modules"), join(target, "node_modules"), process.platform === "win32" ? "junction" : "dir");
|
|
213
|
+
return target;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function isolatedEnvironment(home) {
|
|
217
|
+
mkdirSync(home, { recursive: true });
|
|
218
|
+
const env = { ...process.env };
|
|
219
|
+
for (const key of Object.keys(env)) {
|
|
220
|
+
if (key === "HUNCH_PRIVATE_DIR" || key.endsWith("_API_KEY") || key.endsWith("_TOKEN")) delete env[key];
|
|
221
|
+
}
|
|
222
|
+
Object.assign(env, {
|
|
223
|
+
HOME: home,
|
|
224
|
+
XDG_CONFIG_HOME: join(home, ".config"),
|
|
225
|
+
HUNCH_SYNTH_PROVIDER: "deterministic",
|
|
226
|
+
HUNCH_EMBEDDER: "none",
|
|
227
|
+
GIT_CONFIG_NOSYSTEM: "1",
|
|
228
|
+
GIT_CONFIG_GLOBAL: process.platform === "win32" ? "NUL" : "/dev/null",
|
|
229
|
+
GIT_TERMINAL_PROMPT: "0",
|
|
230
|
+
HTTP_PROXY: "http://127.0.0.1:9",
|
|
231
|
+
HTTPS_PROXY: "http://127.0.0.1:9",
|
|
232
|
+
ALL_PROXY: "http://127.0.0.1:9",
|
|
233
|
+
NO_PROXY: "",
|
|
234
|
+
http_proxy: "http://127.0.0.1:9",
|
|
235
|
+
https_proxy: "http://127.0.0.1:9",
|
|
236
|
+
all_proxy: "http://127.0.0.1:9",
|
|
237
|
+
no_proxy: "",
|
|
238
|
+
});
|
|
239
|
+
return env;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function cliArgs(targetSource, args) {
|
|
243
|
+
return [tsxCli, join(targetSource, "src", "cli", "index.ts"), ...args];
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function runCli(targetSource, fixture, args, env) {
|
|
247
|
+
return run(process.execPath, cliArgs(targetSource, args), { cwd: fixture.root, env });
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function parsePeakRss(stderr) {
|
|
251
|
+
if (process.platform === "darwin") {
|
|
252
|
+
const match = stderr.match(/^\s*(\d+)\s+maximum resident set size\s*$/m);
|
|
253
|
+
if (!match) throw new Error("could not parse macOS /usr/bin/time peak RSS");
|
|
254
|
+
return Number(match[1]);
|
|
255
|
+
}
|
|
256
|
+
if (process.platform === "linux") {
|
|
257
|
+
const match = stderr.match(/Maximum resident set size \(kbytes\):\s*(\d+)/);
|
|
258
|
+
if (!match) throw new Error("could not parse GNU /usr/bin/time peak RSS");
|
|
259
|
+
return Number(match[1]) * 1024;
|
|
260
|
+
}
|
|
261
|
+
throw new Error("peak RSS measurement requires macOS or Linux /usr/bin/time");
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function measureCli(targetSource, fixture, args, env) {
|
|
265
|
+
if (!existsSync("/usr/bin/time")) throw new Error("peak RSS measurement requires /usr/bin/time");
|
|
266
|
+
const timeFlag = process.platform === "darwin" ? "-l" : process.platform === "linux" ? "-v" : null;
|
|
267
|
+
if (!timeFlag) throw new Error("peak RSS measurement is supported on macOS and Linux");
|
|
268
|
+
const started = process.hrtime.bigint();
|
|
269
|
+
const measured = run("/usr/bin/time", [timeFlag, process.execPath, ...cliArgs(targetSource, args)], {
|
|
270
|
+
cwd: fixture.root,
|
|
271
|
+
env,
|
|
272
|
+
});
|
|
273
|
+
const wallMs = Number(process.hrtime.bigint() - started) / 1_000_000;
|
|
274
|
+
return {
|
|
275
|
+
wall_ms: Number(wallMs.toFixed(3)),
|
|
276
|
+
peak_rss_bytes: parsePeakRss(measured.stderr),
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function fixtureSource(index, total) {
|
|
281
|
+
const id = String(index).padStart(3, "0");
|
|
282
|
+
const next = String((index + 1) % total).padStart(3, "0");
|
|
283
|
+
return [
|
|
284
|
+
`import { helper${next} } from "./module-${next}.js";`,
|
|
285
|
+
`export function entry${id}(value: number): number {`,
|
|
286
|
+
` return helper${id}(value) + helper${next}(value);`,
|
|
287
|
+
"}",
|
|
288
|
+
`export function helper${id}(value: number): number {`,
|
|
289
|
+
` return value + ${index + 1};`,
|
|
290
|
+
"}",
|
|
291
|
+
`export function normalize${id}(value: number): number {`,
|
|
292
|
+
` return entry${id}(Math.max(0, value));`,
|
|
293
|
+
"}",
|
|
294
|
+
"",
|
|
295
|
+
].join("\n");
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function initializeGitFixture(root, fileCount) {
|
|
299
|
+
mkdirSync(root, { recursive: true });
|
|
300
|
+
write(root, "package.json", JSON.stringify({
|
|
301
|
+
name: "hunch-md1a-benchmark-fixture",
|
|
302
|
+
private: true,
|
|
303
|
+
type: "module",
|
|
304
|
+
dependencies: { axios: "1.7.0" },
|
|
305
|
+
}, null, 2) + "\n");
|
|
306
|
+
for (let index = 0; index < fileCount; index += 1) {
|
|
307
|
+
write(root, `src/module-${String(index).padStart(3, "0")}.ts`, fixtureSource(index, fileCount));
|
|
308
|
+
}
|
|
309
|
+
git(root, ["init", "-q"]);
|
|
310
|
+
git(root, ["config", "user.name", "Hunch Benchmark"]);
|
|
311
|
+
git(root, ["config", "user.email", "benchmark@example.invalid"]);
|
|
312
|
+
git(root, ["add", "package.json", "src"]);
|
|
313
|
+
git(root, ["commit", "-qm", "fixture: deterministic MD-1a benchmark corpus"], {
|
|
314
|
+
GIT_AUTHOR_NAME: "Hunch Benchmark",
|
|
315
|
+
GIT_AUTHOR_EMAIL: "benchmark@example.invalid",
|
|
316
|
+
GIT_AUTHOR_DATE: FIXTURE_GIT_DATE,
|
|
317
|
+
GIT_COMMITTER_NAME: "Hunch Benchmark",
|
|
318
|
+
GIT_COMMITTER_EMAIL: "benchmark@example.invalid",
|
|
319
|
+
GIT_COMMITTER_DATE: FIXTURE_GIT_DATE,
|
|
320
|
+
});
|
|
321
|
+
return git(root, ["rev-parse", "HEAD"]);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function initializeMemoryHome(hunchDir) {
|
|
325
|
+
mkdirSync(hunchDir, { recursive: true });
|
|
326
|
+
writeHunchJson(join(hunchDir, "manifest.json"), { schema_version: 2 });
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function correctionRecord(index) {
|
|
330
|
+
const id = String(index).padStart(3, "0");
|
|
331
|
+
return {
|
|
332
|
+
id: `con_bench_${id}`,
|
|
333
|
+
type: "architecture",
|
|
334
|
+
statement: `Never import axios in benchmark module ${id}.`,
|
|
335
|
+
scope: [`src/module-${id}.ts`],
|
|
336
|
+
severity: "blocking",
|
|
337
|
+
enforcement: "advisory_v1",
|
|
338
|
+
match: null,
|
|
339
|
+
forbids: { deps: ["axios"], symbols: [], patterns: [] },
|
|
340
|
+
rationale: "Deterministic MD-1a benchmark correction.",
|
|
341
|
+
source_decision: null,
|
|
342
|
+
violations: [],
|
|
343
|
+
status: "active",
|
|
344
|
+
valid_from: FIXTURE_DATE,
|
|
345
|
+
valid_to: null,
|
|
346
|
+
provenance: {
|
|
347
|
+
source: "human_confirmed",
|
|
348
|
+
confidence: 1,
|
|
349
|
+
evidence: ["md1a-performance-fixture"],
|
|
350
|
+
last_verified: FIXTURE_DATE,
|
|
351
|
+
},
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function countJsonFiles(dir) {
|
|
356
|
+
return existsSync(dir) ? readdirSync(dir).filter((file) => file.endsWith(".json")).length : 0;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
function policyStates(hunchDir) {
|
|
360
|
+
const dir = join(hunchDir, "policies");
|
|
361
|
+
if (!existsSync(dir)) return [];
|
|
362
|
+
return readdirSync(dir).filter((file) => file.endsWith(".json")).map((file) => {
|
|
363
|
+
const value = JSON.parse(readFileSync(join(dir, file), "utf8"));
|
|
364
|
+
return value.state;
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
function assertNoAuthority(fixture) {
|
|
369
|
+
for (const hunchDir of [fixture.publicHunch, fixture.privateHunch].filter(Boolean)) {
|
|
370
|
+
const active = policyStates(hunchDir).filter((state) => state === "active_advisory" || state === "active_blocking");
|
|
371
|
+
if (active.length) throw new Error("benchmark setup unexpectedly activated a correction-derived policy");
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
function createFixture(temp, variant, targetSource, scenario, fileCount, selectedOperations, env) {
|
|
376
|
+
const slug = `${variant}-${scenario.home}-${scenario.active_corrections}`;
|
|
377
|
+
const root = join(temp, "fixtures", slug, "repository");
|
|
378
|
+
const publicHunch = join(root, ".hunch");
|
|
379
|
+
const commit = initializeGitFixture(root, fileCount);
|
|
380
|
+
initializeMemoryHome(publicHunch);
|
|
381
|
+
|
|
382
|
+
let privateHunch = null;
|
|
383
|
+
if (scenario.home === "split_private") {
|
|
384
|
+
const overlayRoot = join(temp, "fixtures", slug, "overlay");
|
|
385
|
+
mkdirSync(overlayRoot, { recursive: true });
|
|
386
|
+
git(overlayRoot, ["init", "-q"]);
|
|
387
|
+
privateHunch = join(overlayRoot, ".hunch");
|
|
388
|
+
initializeMemoryHome(privateHunch);
|
|
389
|
+
writeHunchJson(join(publicHunch, "local.json"), {
|
|
390
|
+
privateDir: privateHunch,
|
|
391
|
+
mode: "private",
|
|
392
|
+
autoCommit: false,
|
|
393
|
+
});
|
|
394
|
+
} else {
|
|
395
|
+
writeHunchJson(join(publicHunch, "local.json"), { autoCommit: false });
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
const fixture = { root, publicHunch, privateHunch, commit };
|
|
399
|
+
runCli(targetSource, fixture, ["index"], env);
|
|
400
|
+
runCli(targetSource, fixture, [
|
|
401
|
+
"sync", "HEAD", "--quiet", "--no-commit",
|
|
402
|
+
...(scenario.home === "split_private" ? ["--private"] : []),
|
|
403
|
+
], env);
|
|
404
|
+
|
|
405
|
+
const correctionHome = privateHunch ?? publicHunch;
|
|
406
|
+
mkdirSync(join(correctionHome, "constraints"), { recursive: true });
|
|
407
|
+
for (let index = 0; index < scenario.active_corrections; index += 1) {
|
|
408
|
+
const correction = correctionRecord(index);
|
|
409
|
+
writeHunchJson(join(correctionHome, "constraints", `${correction.id}.json`), correction);
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
const warmIndex = runCli(targetSource, fixture, ["index"], env);
|
|
413
|
+
if (selectedOperations.has("sync")) {
|
|
414
|
+
runCli(targetSource, fixture, [
|
|
415
|
+
"sync", "HEAD", "--quiet", "--no-commit",
|
|
416
|
+
...(scenario.home === "split_private" ? ["--private"] : []),
|
|
417
|
+
], env);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
const correctionCount = countJsonFiles(join(correctionHome, "constraints"));
|
|
421
|
+
if (correctionCount !== scenario.active_corrections) {
|
|
422
|
+
throw new Error(`fixture ${slug} has ${correctionCount} corrections, expected ${scenario.active_corrections}`);
|
|
423
|
+
}
|
|
424
|
+
const policyCount = countJsonFiles(join(correctionHome, "policies"));
|
|
425
|
+
if (scenario.home === "split_private") {
|
|
426
|
+
const leakedPublicRecords = ["constraints", "policies", "plans", "proofs", "evidence"]
|
|
427
|
+
.reduce((total, kind) => total + countJsonFiles(join(publicHunch, kind)), 0);
|
|
428
|
+
if (leakedPublicRecords !== 0) {
|
|
429
|
+
throw new Error(`split-private fixture ${slug} leaked ${leakedPublicRecords} private correction artifact(s) into the public home`);
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
if (variant === "current" && scenario.active_corrections > 0) {
|
|
433
|
+
if (!warmIndex.stdout.includes("correction reviews:")) throw new Error(`current fixture ${slug} did not observe automatic correction retry`);
|
|
434
|
+
if (policyCount !== scenario.active_corrections) {
|
|
435
|
+
throw new Error(`current fixture ${slug} proved ${policyCount} policies, expected ${scenario.active_corrections}`);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
if (variant === "baseline" && policyCount !== 0) throw new Error(`baseline fixture ${slug} unexpectedly created MD-1a policies`);
|
|
439
|
+
assertNoAuthority(fixture);
|
|
440
|
+
const remotes = git(root, ["remote"]).split("\n").filter(Boolean).length
|
|
441
|
+
+ (privateHunch ? git(dirname(privateHunch), ["remote"]).split("\n").filter(Boolean).length : 0);
|
|
442
|
+
if (remotes !== 0) throw new Error(`fixture ${slug} unexpectedly has ${remotes} git remote(s)`);
|
|
443
|
+
return { ...fixture, prepared_policies: policyCount, remotes };
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
function operationArgs(scenario) {
|
|
447
|
+
if (scenario.operation === "index") return ["index"];
|
|
448
|
+
return [
|
|
449
|
+
"sync", "HEAD", "--quiet", "--no-commit",
|
|
450
|
+
...(scenario.home === "split_private" ? ["--private"] : []),
|
|
451
|
+
];
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
function ratio(current, baseline) {
|
|
455
|
+
return baseline === 0 ? null : Number((current / baseline).toFixed(4));
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
function comparisonFor(scenario, baselineCase, currentCase) {
|
|
459
|
+
const measure = (field) => ({
|
|
460
|
+
p50_delta: Number((currentCase.summary[field].p50 - baselineCase.summary[field].p50).toFixed(3)),
|
|
461
|
+
p50_ratio: ratio(currentCase.summary[field].p50, baselineCase.summary[field].p50),
|
|
462
|
+
p95_delta: Number((currentCase.summary[field].p95 - baselineCase.summary[field].p95).toFixed(3)),
|
|
463
|
+
p95_ratio: ratio(currentCase.summary[field].p95, baselineCase.summary[field].p95),
|
|
464
|
+
});
|
|
465
|
+
return {
|
|
466
|
+
scenario: scenario.id,
|
|
467
|
+
baseline_case: baselineCase.id,
|
|
468
|
+
current_case: currentCase.id,
|
|
469
|
+
wall_ms: measure("wall_ms"),
|
|
470
|
+
peak_rss_bytes: measure("peak_rss_bytes"),
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
function atomicWrite(file, value) {
|
|
475
|
+
const target = resolve(process.cwd(), file);
|
|
476
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
477
|
+
const temporary = `${target}.tmp-${process.pid}`;
|
|
478
|
+
writeFileSync(temporary, JSON.stringify(value, null, 2) + "\n");
|
|
479
|
+
renameSync(temporary, target);
|
|
480
|
+
return target;
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
export async function runBenchmark(options) {
|
|
484
|
+
if (!["darwin", "linux"].includes(process.platform)) {
|
|
485
|
+
throw new Error("MD-1a peak-RSS benchmarking currently requires macOS or Linux");
|
|
486
|
+
}
|
|
487
|
+
if (!existsSync(tsxCli)) throw new Error(`tsx runner not found at ${tsxCli}; run npm install first`);
|
|
488
|
+
const scenarios = selectScenarios(options);
|
|
489
|
+
if (!scenarios.length) throw new Error("no benchmark scenarios selected");
|
|
490
|
+
const temporary = mkdtempSync(join(tmpdir(), "hunch-md1a-benchmark-"));
|
|
491
|
+
let receipt;
|
|
492
|
+
try {
|
|
493
|
+
const baselineSource = baselineCheckout(temporary);
|
|
494
|
+
const environment = isolatedEnvironment(join(temporary, "home"));
|
|
495
|
+
const variants = {
|
|
496
|
+
baseline: { target: baselineSource },
|
|
497
|
+
current: { target: projectRoot },
|
|
498
|
+
};
|
|
499
|
+
const scenarioFamilies = [...new Map(scenarios.map((scenario) => [
|
|
500
|
+
`${scenario.home}.${scenario.active_corrections}`,
|
|
501
|
+
{ home: scenario.home, active_corrections: scenario.active_corrections },
|
|
502
|
+
])).values()];
|
|
503
|
+
const fixtures = new Map();
|
|
504
|
+
let fixtureRemotes = 0;
|
|
505
|
+
for (const family of scenarioFamilies) {
|
|
506
|
+
const operations = new Set(scenarios
|
|
507
|
+
.filter((scenario) => scenario.home === family.home && scenario.active_corrections === family.active_corrections)
|
|
508
|
+
.map((scenario) => scenario.operation));
|
|
509
|
+
for (const [variant, details] of Object.entries(variants)) {
|
|
510
|
+
const fixture = createFixture(temporary, variant, details.target, family, options.files, operations, environment);
|
|
511
|
+
fixtures.set(`${variant}.${family.home}.${family.active_corrections}`, fixture);
|
|
512
|
+
fixtureRemotes += fixture.remotes;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
const cases = [];
|
|
517
|
+
const comparisons = [];
|
|
518
|
+
for (const scenario of scenarios) {
|
|
519
|
+
const samples = { baseline: [], current: [] };
|
|
520
|
+
for (let sample = 0; sample < options.samples; sample += 1) {
|
|
521
|
+
const order = sample % 2 === 0 ? ["baseline", "current"] : ["current", "baseline"];
|
|
522
|
+
for (const variant of order) {
|
|
523
|
+
const fixture = fixtures.get(`${variant}.${scenario.home}.${scenario.active_corrections}`);
|
|
524
|
+
samples[variant].push(measureCli(variants[variant].target, fixture, operationArgs(scenario), environment));
|
|
525
|
+
assertNoAuthority(fixture);
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
const scenarioCases = {};
|
|
529
|
+
for (const variant of ["baseline", "current"]) {
|
|
530
|
+
const fixture = fixtures.get(`${variant}.${scenario.home}.${scenario.active_corrections}`);
|
|
531
|
+
const entry = {
|
|
532
|
+
id: `case_${variant}_${scenario.id.replaceAll(".", "_")}`,
|
|
533
|
+
variant,
|
|
534
|
+
operation: scenario.operation === "sync" ? "noop_sync" : "index",
|
|
535
|
+
home: scenario.home,
|
|
536
|
+
active_corrections: scenario.active_corrections,
|
|
537
|
+
fixture_commit: fixture.commit,
|
|
538
|
+
fixture_files: options.files,
|
|
539
|
+
warmup_runs: 1,
|
|
540
|
+
feature: {
|
|
541
|
+
correction_retry: variant === "current" ? "automatic" : "unavailable",
|
|
542
|
+
retry_state: scenario.active_corrections > 0 && variant === "current" ? "already_proved" : "empty_or_not_supported",
|
|
543
|
+
prepared_non_authoritative_policies: fixture.prepared_policies,
|
|
544
|
+
},
|
|
545
|
+
samples: samples[variant],
|
|
546
|
+
summary: summarizeSamples(samples[variant]),
|
|
547
|
+
};
|
|
548
|
+
cases.push(entry);
|
|
549
|
+
scenarioCases[variant] = entry;
|
|
550
|
+
}
|
|
551
|
+
comparisons.push(comparisonFor(scenario, scenarioCases.baseline, scenarioCases.current));
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
const currentCommit = git(projectRoot, ["rev-parse", "HEAD"]);
|
|
555
|
+
const body = {
|
|
556
|
+
schema: BENCHMARK_SCHEMA,
|
|
557
|
+
generated_at: new Date().toISOString(),
|
|
558
|
+
baseline: {
|
|
559
|
+
ref: `main@${BENCHMARK_BASELINE_REF.slice(0, 7)}`,
|
|
560
|
+
commit: BENCHMARK_BASELINE_REF,
|
|
561
|
+
source_hash: sourceHash(baselineSource),
|
|
562
|
+
},
|
|
563
|
+
current: {
|
|
564
|
+
commit: currentCommit,
|
|
565
|
+
worktree_changes: git(projectRoot, ["status", "--porcelain", "--untracked-files=all"]) !== "",
|
|
566
|
+
source_hash: sourceHash(projectRoot),
|
|
567
|
+
},
|
|
568
|
+
environment: {
|
|
569
|
+
node: process.version,
|
|
570
|
+
platform: process.platform,
|
|
571
|
+
arch: process.arch,
|
|
572
|
+
cpu_model: cpus()[0]?.model ?? "unknown",
|
|
573
|
+
cpu_count: cpus().length,
|
|
574
|
+
total_memory_bytes: totalmem(),
|
|
575
|
+
timing: "process.hrtime.bigint",
|
|
576
|
+
peak_rss: process.platform === "darwin" ? "/usr/bin/time -l (bytes)" : "/usr/bin/time -v (KiB normalized to bytes)",
|
|
577
|
+
},
|
|
578
|
+
configuration: {
|
|
579
|
+
samples_per_case: options.samples,
|
|
580
|
+
bounded_many: options.many,
|
|
581
|
+
fixture_files: options.files,
|
|
582
|
+
runner_hash: `sha256:${sha256(readFileSync(fileURLToPath(import.meta.url)))}`,
|
|
583
|
+
percentile_method: "nearest-rank",
|
|
584
|
+
sample_order: "baseline/current alternated by sample",
|
|
585
|
+
timed_scope: "real source CLI after fixture setup and one warm-up",
|
|
586
|
+
},
|
|
587
|
+
safety: {
|
|
588
|
+
network_access: "disabled-by-construction",
|
|
589
|
+
synthesis_provider: "deterministic",
|
|
590
|
+
proxy_route: "closed-loopback:9",
|
|
591
|
+
fixture_remotes: fixtureRemotes,
|
|
592
|
+
authority_grants: 0,
|
|
593
|
+
private_data_in_public_fixture: false,
|
|
594
|
+
},
|
|
595
|
+
cases,
|
|
596
|
+
comparisons,
|
|
597
|
+
};
|
|
598
|
+
const contentHash = `sha256:${sha256(JSON.stringify(stable(body)))}`;
|
|
599
|
+
receipt = {
|
|
600
|
+
id: `md1bench_${contentHash.slice("sha256:".length, "sha256:".length + 12)}`,
|
|
601
|
+
content_hash: contentHash,
|
|
602
|
+
...body,
|
|
603
|
+
};
|
|
604
|
+
} finally {
|
|
605
|
+
if (options.keepTemp) process.stderr.write(`MD-1a benchmark fixtures retained at ${temporary}\n`);
|
|
606
|
+
else rmSync(temporary, { recursive: true, force: true });
|
|
607
|
+
}
|
|
608
|
+
if (options.output) atomicWrite(options.output, receipt);
|
|
609
|
+
return receipt;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
async function main() {
|
|
613
|
+
try {
|
|
614
|
+
const options = parseBenchmarkArgs(process.argv.slice(2));
|
|
615
|
+
if (options.help) {
|
|
616
|
+
process.stdout.write(`${help()}\n`);
|
|
617
|
+
return;
|
|
618
|
+
}
|
|
619
|
+
const receipt = await runBenchmark(options);
|
|
620
|
+
process.stdout.write(`${JSON.stringify(receipt, null, 2)}\n`);
|
|
621
|
+
} catch (error) {
|
|
622
|
+
process.stderr.write(`MD-1a benchmark failed: ${error.message}\n`);
|
|
623
|
+
process.exitCode = 1;
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
const invoked = process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
|
|
628
|
+
if (invoked) await main();
|