@tachikomagundam/abathur 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +29 -0
- package/.github/workflows/publish.yml +74 -0
- package/LICENSE +21 -0
- package/README.md +461 -0
- package/config/abathur.jsonc +17 -0
- package/config/genomes/historian.example.jsonc +124 -0
- package/dist/bench/adapter.js +201 -0
- package/dist/bench/fixture-probe.js +92 -0
- package/dist/bench/fixture-support.js +173 -0
- package/dist/bench/fixture.js +236 -0
- package/dist/bench/toy.js +152 -0
- package/dist/cli.js +110 -0
- package/dist/commands/bundle.js +79 -0
- package/dist/commands/genome.js +94 -0
- package/dist/commands/graft.js +71 -0
- package/dist/commands/kernel.js +47 -0
- package/dist/commands/promote.js +25 -0
- package/dist/commands/run.js +145 -0
- package/dist/commands/self-eval.js +240 -0
- package/dist/commands/status.js +186 -0
- package/dist/commands/tombstone.js +72 -0
- package/dist/config.js +161 -0
- package/dist/core/bundle-common.js +119 -0
- package/dist/core/bundle-export.js +212 -0
- package/dist/core/bundle-inspect.js +143 -0
- package/dist/core/bundle-manifest.js +105 -0
- package/dist/core/bundle-mask.js +75 -0
- package/dist/core/bundle-tar.js +240 -0
- package/dist/core/bundle.js +9 -0
- package/dist/core/evolve/brief.js +45 -0
- package/dist/core/evolve/candidate.js +140 -0
- package/dist/core/evolve/child-track.js +197 -0
- package/dist/core/evolve/friction.js +150 -0
- package/dist/core/evolve/reflect.js +191 -0
- package/dist/core/evolve/run-bench.js +170 -0
- package/dist/core/evolve/run-friction.js +63 -0
- package/dist/core/evolve/run-loop.js +282 -0
- package/dist/core/evolve/run-plan.js +39 -0
- package/dist/core/evolve/run-rows.js +145 -0
- package/dist/core/evolve/self-overlay.js +213 -0
- package/dist/core/evolve/self-snapshot.js +170 -0
- package/dist/core/evolve/stub-mutators.mjs +105 -0
- package/dist/core/evolve/udiff.js +189 -0
- package/dist/core/genome-paths.js +76 -0
- package/dist/core/genome.js +176 -0
- package/dist/core/glob.js +106 -0
- package/dist/core/graft-gates.js +184 -0
- package/dist/core/graft-rebench.js +187 -0
- package/dist/core/graft-support.js +181 -0
- package/dist/core/graft.js +218 -0
- package/dist/core/ids.js +154 -0
- package/dist/core/incumbent.js +46 -0
- package/dist/core/kernel.js +112 -0
- package/dist/core/ledger.js +198 -0
- package/dist/core/locks.js +172 -0
- package/dist/core/promote.js +119 -0
- package/dist/core/snapshot.js +61 -0
- package/dist/core/spec.js +178 -0
- package/dist/core/stats-math.js +102 -0
- package/dist/core/stats-pareto.js +57 -0
- package/dist/core/stats.js +184 -0
- package/dist/core/worktree.js +190 -0
- package/dist/exit.js +32 -0
- package/dist/genomes/toy-smoke/genome.jsonc +30 -0
- package/dist/genomes/toy-smoke/grader.mjs +61 -0
- package/dist/genomes/toy-smoke/init.mjs +63 -0
- package/dist/genomes/toy-smoke/units/add.mjs +17 -0
- package/dist/genomes/toy-smoke/units/explode.mjs +4 -0
- package/dist/genomes/toy-smoke/units/hang.mjs +16 -0
- package/dist/genomes/toy-smoke/units/mul.mjs +16 -0
- package/dist/genomes/toy-smoke/units/mutate.mjs +18 -0
- package/dist/genomes/toy-smoke/units/sub.mjs +16 -0
- package/dist/jsonc.js +77 -0
- package/dist/out.js +5 -0
- package/dist/test/bench-adapter.test.js +33 -0
- package/dist/test/bench-fixture.test.js +407 -0
- package/dist/test/bench-toy.test.js +251 -0
- package/dist/test/bundle.test.js +659 -0
- package/dist/test/config.test.js +185 -0
- package/dist/test/d7-gate.test.js +56 -0
- package/dist/test/fixture-loop.test.js +267 -0
- package/dist/test/fixtures/friction-writer.js +16 -0
- package/dist/test/fixtures-historian.js +82 -0
- package/dist/test/fixtures-self.js +143 -0
- package/dist/test/fixtures-wt.js +64 -0
- package/dist/test/friction.test.js +398 -0
- package/dist/test/genome.test.js +453 -0
- package/dist/test/git.test.js +69 -0
- package/dist/test/graft.test.js +567 -0
- package/dist/test/historian-genome.test.js +134 -0
- package/dist/test/historian-grader-io.test.js +148 -0
- package/dist/test/historian-grader.test.js +209 -0
- package/dist/test/ids.test.js +116 -0
- package/dist/test/include-val.test.js +120 -0
- package/dist/test/ledger-lock.test.js +99 -0
- package/dist/test/ledger.test.js +102 -0
- package/dist/test/promote.test.js +394 -0
- package/dist/test/reflect.test.js +410 -0
- package/dist/test/run-loop.test.js +433 -0
- package/dist/test/self-snapshot.test.js +328 -0
- package/dist/test/snapshot.test.js +86 -0
- package/dist/test/stats.test.js +423 -0
- package/dist/test/stub-mutators.test.js +17 -0
- package/dist/test/testutil.js +30 -0
- package/dist/test/worktree.test.js +198 -0
- package/dist/util/freeze.js +30 -0
- package/dist/util/git.js +85 -0
- package/docs/federation.md +184 -0
- package/docs/immutable-kernel.md +87 -0
- package/graders/historian/grader-core.d.mts +53 -0
- package/graders/historian/grader-core.mjs +276 -0
- package/graders/historian/grader-support.d.mts +57 -0
- package/graders/historian/grader-support.mjs +137 -0
- package/graders/historian/grader.mjs +113 -0
- package/graders/historian/mutate.sh +114 -0
- package/graders/historian/reset-sandbox.sh +60 -0
- package/graders/historian/run-scenario.sh +49 -0
- package/graders/historian/seed-wrapped.sh +32 -0
- package/package.json +42 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// `abathur graft <bundle.tgz> --genome <label>` — thin CLI edge (plan todo 13):
|
|
2
|
+
// flag parsing + registry lookup + stdout funnel. All gates, queue mechanics
|
|
3
|
+
// and the local re-bench live in src/core/graft*.ts so the flow stays unit
|
|
4
|
+
// testable and the CLI never touches process.
|
|
5
|
+
import { homedir } from "node:os";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { resolveConfigDir } from "../config.js";
|
|
8
|
+
import { readRegistry } from "../core/genome.js";
|
|
9
|
+
import { graftBundle } from "../core/graft.js";
|
|
10
|
+
import { cannotAnswer } from "../exit.js";
|
|
11
|
+
import { writeStdout } from "../out.js";
|
|
12
|
+
const USAGE = "usage: abathur graft <bundle.tgz> --genome <label>";
|
|
13
|
+
function parseGraftFlags(args) {
|
|
14
|
+
let bundlePath = null;
|
|
15
|
+
let label = null;
|
|
16
|
+
for (let i = 0; i < args.length; i += 1) {
|
|
17
|
+
const arg = args[i];
|
|
18
|
+
if (arg === "--genome") {
|
|
19
|
+
const raw = args[(i += 1)];
|
|
20
|
+
if (raw === undefined || raw.length === 0 || raw.startsWith("--"))
|
|
21
|
+
cannotAnswer("graft: --genome <label> requires a value", USAGE);
|
|
22
|
+
label = raw;
|
|
23
|
+
}
|
|
24
|
+
else if (arg.startsWith("--")) {
|
|
25
|
+
cannotAnswer(`graft: unknown flag ${arg}`, `${USAGE}\n no bypass flag exists — quarantine and pending-bench are decisions, not warnings`);
|
|
26
|
+
}
|
|
27
|
+
else if (bundlePath !== null) {
|
|
28
|
+
cannotAnswer(`graft: unexpected extra argument '${arg}'`, USAGE);
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
bundlePath = arg;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (bundlePath === null)
|
|
35
|
+
cannotAnswer("graft: <bundle.tgz> is required", USAGE);
|
|
36
|
+
if (label === null)
|
|
37
|
+
cannotAnswer("graft: --genome <label> is required", USAGE);
|
|
38
|
+
return { bundlePath, label };
|
|
39
|
+
}
|
|
40
|
+
/** Registry lookup WITHOUT run.ts's resolveUniqueEntry so an unregistered
|
|
41
|
+
* label can reach the pending-bench queue path instead of exiting 2. */
|
|
42
|
+
function entryForLabel(configDir, label) {
|
|
43
|
+
const matches = readRegistry(configDir).entries.filter((entry) => entry.label === label);
|
|
44
|
+
if (matches.length > 1) {
|
|
45
|
+
cannotAnswer(`graft: label '${label}' is ambiguous — ${String(matches.length)} genomes share it: ${matches
|
|
46
|
+
.map((e) => `${e.fingerprint} @ ${e.spec.repoPath}`)
|
|
47
|
+
.join(", ")}`, "give the specs distinct labels; the fingerprint is the identity");
|
|
48
|
+
}
|
|
49
|
+
return matches[0] ?? null;
|
|
50
|
+
}
|
|
51
|
+
async function runGraft(context) {
|
|
52
|
+
const flags = parseGraftFlags(context.args);
|
|
53
|
+
const configDir = resolveConfigDir();
|
|
54
|
+
const outcome = await graftBundle({
|
|
55
|
+
entry: entryForLabel(configDir, flags.label),
|
|
56
|
+
configDir,
|
|
57
|
+
bundlePath: path.resolve(flags.bundlePath),
|
|
58
|
+
genomeLabel: flags.label,
|
|
59
|
+
home: homedir(),
|
|
60
|
+
opencodeBin: context.loaded.config.opencodeBin,
|
|
61
|
+
env: process.env,
|
|
62
|
+
});
|
|
63
|
+
for (const line of outcome.lines)
|
|
64
|
+
writeStdout(line);
|
|
65
|
+
return outcome.exitCode;
|
|
66
|
+
}
|
|
67
|
+
export const graftCommand = {
|
|
68
|
+
name: "graft",
|
|
69
|
+
summary: "import an offline bundle and graft its lineage",
|
|
70
|
+
run: (context) => runGraft(context),
|
|
71
|
+
};
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// `abathur kernel audit <label>` — read-only seal check (todo 4). Promote-time
|
|
2
|
+
// reseal/generation stays with todo 10; this router only reports drift so the
|
|
3
|
+
// validation loop (todo 9) and gate can consume the same auditKernel() result.
|
|
4
|
+
import { resolveConfigDir } from "../config.js";
|
|
5
|
+
import { requireGenomesByLabel } from "../core/genome.js";
|
|
6
|
+
import { auditKernel } from "../core/kernel.js";
|
|
7
|
+
import { EXIT_OK, blocked, cannotAnswer } from "../exit.js";
|
|
8
|
+
import { writeStdout } from "../out.js";
|
|
9
|
+
function kernelAudit(configDir, args) {
|
|
10
|
+
const label = args[0];
|
|
11
|
+
if (label === undefined || label.length === 0) {
|
|
12
|
+
cannotAnswer("kernel audit: missing <label> argument", "usage: abathur kernel audit <label>");
|
|
13
|
+
}
|
|
14
|
+
const scan = requireGenomesByLabel(configDir, label);
|
|
15
|
+
for (const warning of scan.warnings)
|
|
16
|
+
writeStdout(`warning: ${warning}`);
|
|
17
|
+
const failures = [];
|
|
18
|
+
for (const entry of scan.entries) {
|
|
19
|
+
const audit = auditKernel(entry, configDir);
|
|
20
|
+
if (audit.ok) {
|
|
21
|
+
writeStdout(`sealed clean: '${entry.label}' (${entry.fingerprint})`);
|
|
22
|
+
}
|
|
23
|
+
else {
|
|
24
|
+
failures.push(...audit.drifted.map((d) => `${entry.label} (${entry.fingerprint}): ${d.path} ${d.kind}`));
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
if (failures.length > 0) {
|
|
28
|
+
blocked(`kernel audit FAILED for '${label}':\n - ${failures.slice(0, 10).join("\n - ")}` +
|
|
29
|
+
`${failures.length > 10 ? `\n - (+${String(failures.length - 10)} more)` : ""}`, "restore the named files or resolve the drift at promote — 'genome add' cannot reseal");
|
|
30
|
+
}
|
|
31
|
+
return EXIT_OK;
|
|
32
|
+
}
|
|
33
|
+
function runKernel(args) {
|
|
34
|
+
const configDir = resolveConfigDir();
|
|
35
|
+
const [sub, ...rest] = args;
|
|
36
|
+
switch (sub) {
|
|
37
|
+
case "audit":
|
|
38
|
+
return kernelAudit(configDir, rest);
|
|
39
|
+
default:
|
|
40
|
+
return cannotAnswer(`kernel: unknown subcommand '${sub ?? "<none>"}'`, "usage: abathur kernel audit <label> (promote-time reseal arrives with todo 10)");
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
export const kernelCommand = {
|
|
44
|
+
name: "kernel",
|
|
45
|
+
summary: "audit frozen kernel seals against the working tree",
|
|
46
|
+
run: ({ args }) => runKernel(args),
|
|
47
|
+
};
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// `abathur promote <label> <genId>` — the ONLY entry point to the human gate
|
|
2
|
+
// (plan Must NOT: no auto-promote path importable by evolve modules). No --force:
|
|
3
|
+
// every refusal is a bug report, not a flag.
|
|
4
|
+
import { resolveConfigDir } from "../config.js";
|
|
5
|
+
import { EXIT_OK, cannotAnswer } from "../exit.js";
|
|
6
|
+
import { promoteGeneration } from "../core/promote.js";
|
|
7
|
+
import { writeStdout } from "../out.js";
|
|
8
|
+
import { resolveUniqueEntry } from "./run.js";
|
|
9
|
+
const USAGE = "usage: abathur promote <label> <genId>";
|
|
10
|
+
async function runPromote(args) {
|
|
11
|
+
const [label, genId] = args;
|
|
12
|
+
if (label === undefined || genId === undefined || args.length !== 2)
|
|
13
|
+
cannotAnswer(`promote: expected exactly <label> <genId>`, USAGE);
|
|
14
|
+
const configDir = resolveConfigDir();
|
|
15
|
+
const entry = resolveUniqueEntry(configDir, label, "promote");
|
|
16
|
+
const outcome = await promoteGeneration({ entry, configDir, genId });
|
|
17
|
+
for (const line of outcome.lines)
|
|
18
|
+
writeStdout(line);
|
|
19
|
+
return EXIT_OK;
|
|
20
|
+
}
|
|
21
|
+
export const promoteCommand = {
|
|
22
|
+
name: "promote",
|
|
23
|
+
summary: "human gate: promote a nominated gen (ledger-verified, sealed-path re-check, fast-forward + manifest regen)",
|
|
24
|
+
run: ({ args }) => runPromote(args),
|
|
25
|
+
};
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// `abathur run` — thin CLI edge (plan todo 9): flag parsing + registry lookup +
|
|
2
|
+
// stdout funnel only; every gate and the whole evolution loop live in
|
|
3
|
+
// src/core/evolve/run-loop.ts (runEvolution), so the loop stays unit-testable.
|
|
4
|
+
import { resolveConfigDir } from "../config.js";
|
|
5
|
+
import { readRegistry, requireGenomesByLabel } from "../core/genome.js";
|
|
6
|
+
import { appendRunFriction } from "../core/evolve/friction.js";
|
|
7
|
+
import { runEvolution } from "../core/evolve/run-loop.js";
|
|
8
|
+
import { cannotAnswer, ExitSignal } from "../exit.js";
|
|
9
|
+
import { writeStdout } from "../out.js";
|
|
10
|
+
const USAGE = "usage: abathur run --genome <label> [--reps N] [--max-candidates N] [--mutator <template>] [--dry-run] [--include-val]";
|
|
11
|
+
function positiveInt(raw, flag) {
|
|
12
|
+
if (raw === undefined || raw.length === 0 || raw.startsWith("--")) {
|
|
13
|
+
cannotAnswer(`run: ${flag} requires a positive integer value`, USAGE);
|
|
14
|
+
}
|
|
15
|
+
const value = Number(raw);
|
|
16
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
17
|
+
cannotAnswer(`run: ${flag} expects a positive integer, got '${raw}'`, USAGE);
|
|
18
|
+
}
|
|
19
|
+
return value;
|
|
20
|
+
}
|
|
21
|
+
export function parseRunFlags(args) {
|
|
22
|
+
let label = null;
|
|
23
|
+
let reps = null;
|
|
24
|
+
let maxCandidates = null;
|
|
25
|
+
let mutator = null;
|
|
26
|
+
let dryRun = false;
|
|
27
|
+
let includeVal = false;
|
|
28
|
+
for (let i = 0; i < args.length; i += 1) {
|
|
29
|
+
const arg = args[i];
|
|
30
|
+
const next = () => args[(i += 1)];
|
|
31
|
+
switch (arg) {
|
|
32
|
+
case "--genome": {
|
|
33
|
+
const raw = next();
|
|
34
|
+
if (raw === undefined || raw.length === 0 || raw.startsWith("--")) {
|
|
35
|
+
cannotAnswer("run: --genome <label> requires a value", USAGE);
|
|
36
|
+
}
|
|
37
|
+
label = raw;
|
|
38
|
+
break;
|
|
39
|
+
}
|
|
40
|
+
case "--reps":
|
|
41
|
+
reps = positiveInt(next(), "--reps");
|
|
42
|
+
break;
|
|
43
|
+
case "--max-candidates":
|
|
44
|
+
maxCandidates = positiveInt(next(), "--max-candidates");
|
|
45
|
+
break;
|
|
46
|
+
case "--mutator": {
|
|
47
|
+
const raw = next();
|
|
48
|
+
if (raw === undefined || raw.length === 0 || raw.startsWith("--"))
|
|
49
|
+
cannotAnswer("run: --mutator <template> requires a value", USAGE);
|
|
50
|
+
mutator = raw;
|
|
51
|
+
break;
|
|
52
|
+
}
|
|
53
|
+
case "--dry-run":
|
|
54
|
+
dryRun = true;
|
|
55
|
+
break;
|
|
56
|
+
case "--include-val":
|
|
57
|
+
includeVal = true;
|
|
58
|
+
break;
|
|
59
|
+
default:
|
|
60
|
+
cannotAnswer(`run: unknown flag ${arg}`, USAGE);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (label === null)
|
|
64
|
+
cannotAnswer("run: --genome <label> is required", USAGE);
|
|
65
|
+
return { label, reps, maxCandidates, mutator, dryRun, includeVal };
|
|
66
|
+
}
|
|
67
|
+
/** Labels are free (todo 4): a command needs ONE genome, so duplicates are an explicit exit 2. */
|
|
68
|
+
export function resolveUniqueEntry(configDir, label, who = "run") {
|
|
69
|
+
const scan = requireGenomesByLabel(configDir, label);
|
|
70
|
+
if (scan.entries.length > 1) {
|
|
71
|
+
cannotAnswer(`${who}: label '${label}' is ambiguous — ${String(scan.entries.length)} genomes share it: ${scan.entries
|
|
72
|
+
.map((e) => `${e.fingerprint} @ ${e.spec.repoPath}`)
|
|
73
|
+
.join(", ")}`, "give the specs distinct labels; the fingerprint is the identity");
|
|
74
|
+
}
|
|
75
|
+
const entry = scan.entries[0];
|
|
76
|
+
if (entry === undefined)
|
|
77
|
+
cannotAnswer(`${who}: no registered genome with label '${label}'`);
|
|
78
|
+
return entry;
|
|
79
|
+
}
|
|
80
|
+
const ZERO_COUNTS = { applied: 0, rejected: 0, benched: 0, inconclusive: 0, nominated: 0, timeouts: 0, reaped: 0 };
|
|
81
|
+
async function runRun(context) {
|
|
82
|
+
const flags = parseRunFlags(context.args);
|
|
83
|
+
const configDir = resolveConfigDir();
|
|
84
|
+
for (const warning of readRegistry(configDir).warnings)
|
|
85
|
+
writeStdout(`warning: ${warning}`);
|
|
86
|
+
const entry = resolveUniqueEntry(configDir, flags.label);
|
|
87
|
+
const warnings = [];
|
|
88
|
+
const frictionSink = (input) => {
|
|
89
|
+
try {
|
|
90
|
+
appendRunFriction(configDir, input);
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
warnings.push(`friction: append failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
// Friction is the self-evolution SIGNAL channel (todo 11): every real run
|
|
97
|
+
// books one structured run-summary record, and every refusal that happens
|
|
98
|
+
// after the registry resolved books a cli-error record — in both cases the
|
|
99
|
+
// queue append can never break or change the run's own exit status.
|
|
100
|
+
let outcome;
|
|
101
|
+
try {
|
|
102
|
+
outcome = await runEvolution({
|
|
103
|
+
entry,
|
|
104
|
+
configDir,
|
|
105
|
+
opencodeBin: context.loaded.config.opencodeBin,
|
|
106
|
+
...(flags.reps === null ? {} : { reps: flags.reps }),
|
|
107
|
+
...(flags.maxCandidates === null ? {} : { maxCandidates: flags.maxCandidates }),
|
|
108
|
+
...(flags.mutator === null ? {} : { mutatorCommand: flags.mutator }),
|
|
109
|
+
...(flags.dryRun ? { dryRun: true } : {}),
|
|
110
|
+
...(flags.includeVal ? { includeVal: true } : {}),
|
|
111
|
+
friction: frictionSink,
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
if (error instanceof ExitSignal) {
|
|
116
|
+
try {
|
|
117
|
+
appendRunFriction(configDir, {
|
|
118
|
+
genomeFp: entry.fingerprint,
|
|
119
|
+
cause: "cli-error",
|
|
120
|
+
exit: error.code,
|
|
121
|
+
complete: false,
|
|
122
|
+
counts: ZERO_COUNTS,
|
|
123
|
+
rejected: [],
|
|
124
|
+
units: [],
|
|
125
|
+
reasons: [error.message],
|
|
126
|
+
stall: { budgetTruncated: false, orphanGroups: 0 },
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
// the refusal stands on its own; a dead queue never masks it.
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
throw error;
|
|
134
|
+
}
|
|
135
|
+
for (const line of outcome.lines)
|
|
136
|
+
writeStdout(line);
|
|
137
|
+
for (const warning of warnings)
|
|
138
|
+
writeStdout(warning);
|
|
139
|
+
return outcome.exitCode;
|
|
140
|
+
}
|
|
141
|
+
export const runCommand = {
|
|
142
|
+
name: "run",
|
|
143
|
+
summary: "evolve one genome: observe, mutate, re-bench, select",
|
|
144
|
+
run: (context) => runRun(context),
|
|
145
|
+
};
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
// `abathur self-eval` — REPORT ONLY (plan todo 11c): grade abathur-self
|
|
2
|
+
// candidates by the snapshot-overlay bench (candidate src under the incumbent's
|
|
3
|
+
// trusted tests + golden toy-replay) and print verdicts. It grants NO promote
|
|
4
|
+
// authority — the operator-side `abathur promote` stays the only merge path
|
|
5
|
+
// (structural test in src/test/friction.test.ts forbids importing promote from
|
|
6
|
+
// here), it writes NO ledger rows (reads the ledger raw, status.ts pattern),
|
|
7
|
+
// and its only persistent output is one self-eval friction record appended to
|
|
8
|
+
// the global queue. Single-shot like graft: crash → re-run, nothing to resume.
|
|
9
|
+
//
|
|
10
|
+
// Kernel audit FIRST (same gate order as `run`): a tampered seal means the
|
|
11
|
+
// trusted tree itself is untrustworthy, so scoring refuses with exit 1 BEFORE
|
|
12
|
+
// any snapshot, spawn, or state exists.
|
|
13
|
+
import { resolveConfigDir } from "../config.js";
|
|
14
|
+
import { readRegistry } from "../core/genome.js";
|
|
15
|
+
import { auditKernel } from "../core/kernel.js";
|
|
16
|
+
import { ledgerPath, ledgerRecordSchema } from "../core/ledger.js";
|
|
17
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
18
|
+
import { asReplicates } from "../core/evolve/run-bench.js";
|
|
19
|
+
import { decodeGenerationRecord } from "../core/evolve/run-rows.js";
|
|
20
|
+
import { runExitCode } from "../core/evolve/run-loop.js";
|
|
21
|
+
import { appendRunFriction, scrub } from "../core/evolve/friction.js";
|
|
22
|
+
import { selfBench, selfGuardVerdict } from "../core/evolve/self-snapshot.js";
|
|
23
|
+
import { effectiveRepoPath, isEnvRepoLiteral } from "../core/spec.js";
|
|
24
|
+
import { clampReps, evaluate } from "../core/stats.js";
|
|
25
|
+
import { blocked, cannotAnswer } from "../exit.js";
|
|
26
|
+
import { genId, fingerprint } from "../core/ids.js";
|
|
27
|
+
import { openGenome } from "../core/worktree.js";
|
|
28
|
+
import { writeStdout } from "../out.js";
|
|
29
|
+
import { resolveUniqueEntry } from "./run.js";
|
|
30
|
+
const USAGE = "usage: abathur self-eval [--genome abathur-self] [--gen <genId>]... [--reps N]";
|
|
31
|
+
function parseFlags(args) {
|
|
32
|
+
let label = "abathur-self";
|
|
33
|
+
const gens = [];
|
|
34
|
+
let reps = null;
|
|
35
|
+
let index = 0;
|
|
36
|
+
const next = (flag) => {
|
|
37
|
+
index += 1;
|
|
38
|
+
const raw = args[index];
|
|
39
|
+
if (raw === undefined || raw.length === 0 || raw.startsWith("--"))
|
|
40
|
+
cannotAnswer(`self-eval: ${flag} requires a value`, USAGE);
|
|
41
|
+
return raw;
|
|
42
|
+
};
|
|
43
|
+
while (index < args.length) {
|
|
44
|
+
const arg = args[index];
|
|
45
|
+
if (arg === undefined)
|
|
46
|
+
break;
|
|
47
|
+
switch (arg) {
|
|
48
|
+
case "--genome":
|
|
49
|
+
label = next(arg);
|
|
50
|
+
break;
|
|
51
|
+
case "--gen":
|
|
52
|
+
gens.push(next(arg));
|
|
53
|
+
break;
|
|
54
|
+
case "--reps": {
|
|
55
|
+
const value = Number(next(arg));
|
|
56
|
+
if (!Number.isInteger(value) || value < 1)
|
|
57
|
+
cannotAnswer("self-eval: --reps expects a positive integer", USAGE);
|
|
58
|
+
reps = value;
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
default:
|
|
62
|
+
cannotAnswer(`self-eval: unknown flag ${arg}`, USAGE);
|
|
63
|
+
}
|
|
64
|
+
index += 1;
|
|
65
|
+
}
|
|
66
|
+
return { label, gens, reps };
|
|
67
|
+
}
|
|
68
|
+
function candidateRows(genomeRepo) {
|
|
69
|
+
const file = ledgerPath(genomeRepo);
|
|
70
|
+
if (!existsSync(file))
|
|
71
|
+
return [];
|
|
72
|
+
const out = [];
|
|
73
|
+
const lines = readFileSync(file, "utf8").split("\n");
|
|
74
|
+
if (lines.at(-1) === "")
|
|
75
|
+
lines.pop();
|
|
76
|
+
lines.forEach((line, index) => {
|
|
77
|
+
const value = JSON.parse(line);
|
|
78
|
+
const parsed = ledgerRecordSchema.safeParse(value);
|
|
79
|
+
if (!parsed.success) {
|
|
80
|
+
cannotAnswer(`self-eval: ${file}: line ${String(index + 1)} failed ledger schema validation: ${parsed.error.message}`, "the ledger is append-only truth — repair by hand");
|
|
81
|
+
}
|
|
82
|
+
const record = parsed.data;
|
|
83
|
+
if (record.kind !== "generation_complete" || record.genId === undefined)
|
|
84
|
+
return;
|
|
85
|
+
const data = decodeGenerationRecord(record);
|
|
86
|
+
if (data.source === "candidate" && data.commitSha !== undefined)
|
|
87
|
+
out.push({ genId: record.genId, commit: data.commitSha, data });
|
|
88
|
+
});
|
|
89
|
+
return out;
|
|
90
|
+
}
|
|
91
|
+
function suiteText(res) {
|
|
92
|
+
if (res.suiteRuns.length === 0)
|
|
93
|
+
return `suite ${res.buildStatus === "timeout" ? "BUILD TIMEOUT" : res.buildStatus === "failed" ? "BUILD FAILED" : "no samples"}`;
|
|
94
|
+
return `suite ${res.suiteRuns.map((r) => `${String(r.pass)}/${String(r.tests)}`).join(" ")}`;
|
|
95
|
+
}
|
|
96
|
+
function unitEvidence(res, into) {
|
|
97
|
+
for (const row of res.units) {
|
|
98
|
+
const prior = into.get(row.unitId);
|
|
99
|
+
into.set(row.unitId, {
|
|
100
|
+
unitId: row.unitId,
|
|
101
|
+
split: row.split,
|
|
102
|
+
scores: prior === undefined ? [...row.scores] : [...prior.scores, ...row.scores],
|
|
103
|
+
failures: prior === undefined ? [...row.failures] : [...prior.failures, ...row.failures],
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
async function selfEvalHandler(context) {
|
|
108
|
+
const flags = parseFlags(context.args);
|
|
109
|
+
const configDir = resolveConfigDir();
|
|
110
|
+
for (const warning of readRegistry(configDir).warnings)
|
|
111
|
+
writeStdout(`warning: ${warning}`);
|
|
112
|
+
const entry = resolveUniqueEntry(configDir, flags.label, "self-eval");
|
|
113
|
+
if (!isEnvRepoLiteral(entry.spec.repoPath)) {
|
|
114
|
+
cannotAnswer(`self-eval: '${entry.spec.label}' is not a self genome — repoPath must be the \${ABATHUR_SELF_REPO} literal`, "self-eval grades the harness against its own sealed kernel; see genomes/abathur-self.jsonc");
|
|
115
|
+
}
|
|
116
|
+
const audit = auditKernel(entry, configDir);
|
|
117
|
+
if (!audit.ok) {
|
|
118
|
+
const listed = audit.drifted.map((d) => `${d.path} (${d.kind})`);
|
|
119
|
+
blocked(`self-eval refused: kernel drift for '${entry.spec.label}' (${entry.fingerprint}): ${listed.slice(0, 10).join(", ")}${listed.length > 10 ? ` (+${String(listed.length - 10)} more)` : ""}`, "restore the sealed files or resolve drift via 'abathur kernel' — self-eval never reseals and never scores on a tampered trusted tree");
|
|
120
|
+
}
|
|
121
|
+
const spec = entry.spec;
|
|
122
|
+
const genomeRepo = effectiveRepoPath(spec.repoPath);
|
|
123
|
+
const opened = await openGenome(genomeRepo, [], {});
|
|
124
|
+
const recorded = candidateRows(genomeRepo);
|
|
125
|
+
const byGen = new Map();
|
|
126
|
+
for (const row of recorded)
|
|
127
|
+
byGen.set(row.genId, row); // latest row per genId wins (append order)
|
|
128
|
+
let picked = [...byGen.values()];
|
|
129
|
+
if (flags.gens.length > 0) {
|
|
130
|
+
const wanted = new Set(flags.gens);
|
|
131
|
+
const missing = flags.gens.filter((g) => !byGen.has(g));
|
|
132
|
+
if (missing.length > 0)
|
|
133
|
+
blocked(`self-eval: no candidate generation rows for ${missing.map((m) => `'${m}'`).join(", ")} in '${entry.spec.label}'`, `known candidates: ${[...byGen.keys()].join(", ") || "(none)"}`);
|
|
134
|
+
picked = picked.filter((row) => wanted.has(row.genId));
|
|
135
|
+
}
|
|
136
|
+
if (picked.length === 0) {
|
|
137
|
+
cannotAnswer(`self-eval: no candidate generations recorded for '${entry.spec.label}' — run evolution first`, `abathur run --genome ${entry.spec.label} --mutator <template>`);
|
|
138
|
+
}
|
|
139
|
+
const reps = clampReps(flags.reps ?? undefined, spec.bench.stats.nReps);
|
|
140
|
+
const caps = {
|
|
141
|
+
maxCandidates: spec.budget.maxCandidates,
|
|
142
|
+
maxModelCalls: spec.budget.maxModelCalls,
|
|
143
|
+
maxTokens: spec.budget.maxTokens,
|
|
144
|
+
maxWallS: spec.budget.maxWallS,
|
|
145
|
+
};
|
|
146
|
+
const invRunId = genId(fingerprint({ selfEval: opened.headCommit, at: Date.now() }));
|
|
147
|
+
writeStdout(`self-eval ${entry.spec.label} fp ${entry.fingerprint} incumbent ${opened.headCommit.slice(0, 12)} reps ${String(reps)}`);
|
|
148
|
+
const incumbent = await selfBench({
|
|
149
|
+
spec,
|
|
150
|
+
genomeRepo,
|
|
151
|
+
genomeFp: entry.fingerprint,
|
|
152
|
+
incumbentCommit: opened.headCommit,
|
|
153
|
+
candidateCommit: null,
|
|
154
|
+
genId: `${invRunId}-base`,
|
|
155
|
+
reps,
|
|
156
|
+
env: process.env,
|
|
157
|
+
});
|
|
158
|
+
writeStdout(` baseline: ${suiteText(incumbent)}, replay observed ${incumbent.replayDigest ?? "n/a"} expected ${incumbent.replayExpected}`);
|
|
159
|
+
const evidence = new Map();
|
|
160
|
+
unitEvidence(incumbent, evidence);
|
|
161
|
+
let counters = { candidates: 0, modelCalls: 0, tokens: 0, wallS: incumbent.spent.wallS };
|
|
162
|
+
const verdicts = [];
|
|
163
|
+
const reasons = [];
|
|
164
|
+
for (const row of picked) {
|
|
165
|
+
const res = await selfBench({
|
|
166
|
+
spec,
|
|
167
|
+
genomeRepo,
|
|
168
|
+
genomeFp: entry.fingerprint,
|
|
169
|
+
incumbentCommit: opened.headCommit,
|
|
170
|
+
candidateCommit: row.commit,
|
|
171
|
+
genId: row.genId,
|
|
172
|
+
reps,
|
|
173
|
+
env: process.env,
|
|
174
|
+
});
|
|
175
|
+
unitEvidence(res, evidence);
|
|
176
|
+
counters = { ...counters, candidates: counters.candidates + 1, wallS: counters.wallS + res.spent.wallS };
|
|
177
|
+
const gate = evaluate({
|
|
178
|
+
candidate: { runId: invRunId, units: asReplicates(res.units), counters },
|
|
179
|
+
incumbent: { units: asReplicates(incumbent.units) },
|
|
180
|
+
stats: spec.bench.stats,
|
|
181
|
+
budgetCaps: caps,
|
|
182
|
+
nPairs: Math.max(1, picked.length),
|
|
183
|
+
});
|
|
184
|
+
const verdict = selfGuardVerdict(res.failures, gate.verdict);
|
|
185
|
+
verdicts.push(verdict);
|
|
186
|
+
const gain = gate.gain === null || !Number.isFinite(gate.gain) ? "n/a" : gate.gain.toFixed(4);
|
|
187
|
+
writeStdout(`candidate ${row.data.candidateId ?? row.genId} [gen ${row.genId} tree ${(row.data.treeSha ?? "").slice(0, 12)}]: verdict ${verdict} (gain ${gain}, ${suiteText(res)}, replay observed ${res.replayDigest ?? "n/a"} expected ${res.replayExpected}, overlay ${String(res.overlaid.length)} files, dropped ${String(res.dropped.length)})`);
|
|
188
|
+
if (res.buildStatus !== "ok")
|
|
189
|
+
writeStdout(` build ${res.buildStatus}: ${scrub(res.buildNote, 200)}`);
|
|
190
|
+
if (res.overlaid.length > 0) {
|
|
191
|
+
const shown = res.overlaid.slice(0, 10).join(", ");
|
|
192
|
+
writeStdout(` overlay: ${shown}${res.overlaid.length > 10 ? ` (+${String(res.overlaid.length - 10)} more)` : ""}`);
|
|
193
|
+
}
|
|
194
|
+
for (const reason of ["sealed", "trusted-tests"]) {
|
|
195
|
+
const dropped = res.dropped.filter((d) => d.reason === reason);
|
|
196
|
+
if (dropped.length > 0)
|
|
197
|
+
writeStdout(` dropped (${reason}): ${dropped.map((d) => d.path).join(", ")}`);
|
|
198
|
+
}
|
|
199
|
+
for (const failure of gate.failures)
|
|
200
|
+
writeStdout(` gate: ${failure}`);
|
|
201
|
+
if (verdict !== gate.verdict)
|
|
202
|
+
writeStdout(` guard: self-bench downgraded ${gate.verdict} -> ${verdict} (see overlay-empty/build-timeout rule)`);
|
|
203
|
+
for (const note of res.failures)
|
|
204
|
+
if (note.startsWith("self-bench:"))
|
|
205
|
+
reasons.push(`${row.data.candidateId ?? row.genId}: ${scrub(note, 200)}`);
|
|
206
|
+
}
|
|
207
|
+
const exit = runExitCode(verdicts);
|
|
208
|
+
const timeouts = [...evidence.values()].reduce((n, u) => n + u.failures.filter((f) => f.includes(": run timeout") || f.includes("build timeout")).length, 0);
|
|
209
|
+
const input = {
|
|
210
|
+
genomeFp: entry.fingerprint,
|
|
211
|
+
cause: "self-eval",
|
|
212
|
+
exit,
|
|
213
|
+
complete: incumbent.complete && verdicts.every((v) => v !== "inconclusive"),
|
|
214
|
+
counts: {
|
|
215
|
+
applied: 0,
|
|
216
|
+
rejected: 0,
|
|
217
|
+
benched: picked.length + 1,
|
|
218
|
+
inconclusive: verdicts.filter((v) => v === "inconclusive").length,
|
|
219
|
+
nominated: verdicts.filter((v) => v === "nominated").length,
|
|
220
|
+
timeouts,
|
|
221
|
+
reaped: 0,
|
|
222
|
+
},
|
|
223
|
+
rejected: [],
|
|
224
|
+
units: [...evidence.values()],
|
|
225
|
+
reasons: [`self-eval ${entry.spec.label}: ${String(picked.length)} candidate(s) over snapshot-overlay bench`, ...reasons],
|
|
226
|
+
stall: { budgetTruncated: !incumbent.complete, orphanGroups: 0 },
|
|
227
|
+
};
|
|
228
|
+
try {
|
|
229
|
+
appendRunFriction(configDir, input);
|
|
230
|
+
}
|
|
231
|
+
catch (error) {
|
|
232
|
+
writeStdout(`friction: append failed: ${error instanceof Error ? scrub(error.message, 200) : String(error)}`);
|
|
233
|
+
}
|
|
234
|
+
return exit;
|
|
235
|
+
}
|
|
236
|
+
export const selfEvalCommand = {
|
|
237
|
+
name: "self-eval",
|
|
238
|
+
summary: "evaluate the harness itself (abathur-self genome)",
|
|
239
|
+
run: (context) => selfEvalHandler(context),
|
|
240
|
+
};
|