@tachikomagundam/abathur 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +29 -0
- package/.github/workflows/publish.yml +74 -0
- package/LICENSE +21 -0
- package/README.md +461 -0
- package/config/abathur.jsonc +17 -0
- package/config/genomes/historian.example.jsonc +124 -0
- package/dist/bench/adapter.js +201 -0
- package/dist/bench/fixture-probe.js +92 -0
- package/dist/bench/fixture-support.js +173 -0
- package/dist/bench/fixture.js +236 -0
- package/dist/bench/toy.js +152 -0
- package/dist/cli.js +110 -0
- package/dist/commands/bundle.js +79 -0
- package/dist/commands/genome.js +94 -0
- package/dist/commands/graft.js +71 -0
- package/dist/commands/kernel.js +47 -0
- package/dist/commands/promote.js +25 -0
- package/dist/commands/run.js +145 -0
- package/dist/commands/self-eval.js +240 -0
- package/dist/commands/status.js +186 -0
- package/dist/commands/tombstone.js +72 -0
- package/dist/config.js +161 -0
- package/dist/core/bundle-common.js +119 -0
- package/dist/core/bundle-export.js +212 -0
- package/dist/core/bundle-inspect.js +143 -0
- package/dist/core/bundle-manifest.js +105 -0
- package/dist/core/bundle-mask.js +75 -0
- package/dist/core/bundle-tar.js +240 -0
- package/dist/core/bundle.js +9 -0
- package/dist/core/evolve/brief.js +45 -0
- package/dist/core/evolve/candidate.js +140 -0
- package/dist/core/evolve/child-track.js +197 -0
- package/dist/core/evolve/friction.js +150 -0
- package/dist/core/evolve/reflect.js +191 -0
- package/dist/core/evolve/run-bench.js +170 -0
- package/dist/core/evolve/run-friction.js +63 -0
- package/dist/core/evolve/run-loop.js +282 -0
- package/dist/core/evolve/run-plan.js +39 -0
- package/dist/core/evolve/run-rows.js +145 -0
- package/dist/core/evolve/self-overlay.js +213 -0
- package/dist/core/evolve/self-snapshot.js +170 -0
- package/dist/core/evolve/stub-mutators.mjs +105 -0
- package/dist/core/evolve/udiff.js +189 -0
- package/dist/core/genome-paths.js +76 -0
- package/dist/core/genome.js +176 -0
- package/dist/core/glob.js +106 -0
- package/dist/core/graft-gates.js +184 -0
- package/dist/core/graft-rebench.js +187 -0
- package/dist/core/graft-support.js +181 -0
- package/dist/core/graft.js +218 -0
- package/dist/core/ids.js +154 -0
- package/dist/core/incumbent.js +46 -0
- package/dist/core/kernel.js +112 -0
- package/dist/core/ledger.js +198 -0
- package/dist/core/locks.js +172 -0
- package/dist/core/promote.js +119 -0
- package/dist/core/snapshot.js +61 -0
- package/dist/core/spec.js +178 -0
- package/dist/core/stats-math.js +102 -0
- package/dist/core/stats-pareto.js +57 -0
- package/dist/core/stats.js +184 -0
- package/dist/core/worktree.js +190 -0
- package/dist/exit.js +32 -0
- package/dist/genomes/toy-smoke/genome.jsonc +30 -0
- package/dist/genomes/toy-smoke/grader.mjs +61 -0
- package/dist/genomes/toy-smoke/init.mjs +63 -0
- package/dist/genomes/toy-smoke/units/add.mjs +17 -0
- package/dist/genomes/toy-smoke/units/explode.mjs +4 -0
- package/dist/genomes/toy-smoke/units/hang.mjs +16 -0
- package/dist/genomes/toy-smoke/units/mul.mjs +16 -0
- package/dist/genomes/toy-smoke/units/mutate.mjs +18 -0
- package/dist/genomes/toy-smoke/units/sub.mjs +16 -0
- package/dist/jsonc.js +77 -0
- package/dist/out.js +5 -0
- package/dist/test/bench-adapter.test.js +33 -0
- package/dist/test/bench-fixture.test.js +407 -0
- package/dist/test/bench-toy.test.js +251 -0
- package/dist/test/bundle.test.js +659 -0
- package/dist/test/config.test.js +185 -0
- package/dist/test/d7-gate.test.js +56 -0
- package/dist/test/fixture-loop.test.js +267 -0
- package/dist/test/fixtures/friction-writer.js +16 -0
- package/dist/test/fixtures-historian.js +82 -0
- package/dist/test/fixtures-self.js +143 -0
- package/dist/test/fixtures-wt.js +64 -0
- package/dist/test/friction.test.js +398 -0
- package/dist/test/genome.test.js +453 -0
- package/dist/test/git.test.js +69 -0
- package/dist/test/graft.test.js +567 -0
- package/dist/test/historian-genome.test.js +134 -0
- package/dist/test/historian-grader-io.test.js +148 -0
- package/dist/test/historian-grader.test.js +209 -0
- package/dist/test/ids.test.js +116 -0
- package/dist/test/include-val.test.js +120 -0
- package/dist/test/ledger-lock.test.js +99 -0
- package/dist/test/ledger.test.js +102 -0
- package/dist/test/promote.test.js +394 -0
- package/dist/test/reflect.test.js +410 -0
- package/dist/test/run-loop.test.js +433 -0
- package/dist/test/self-snapshot.test.js +328 -0
- package/dist/test/snapshot.test.js +86 -0
- package/dist/test/stats.test.js +423 -0
- package/dist/test/stub-mutators.test.js +17 -0
- package/dist/test/testutil.js +30 -0
- package/dist/test/worktree.test.js +198 -0
- package/dist/util/freeze.js +30 -0
- package/dist/util/git.js +85 -0
- package/docs/federation.md +184 -0
- package/docs/immutable-kernel.md +87 -0
- package/graders/historian/grader-core.d.mts +53 -0
- package/graders/historian/grader-core.mjs +276 -0
- package/graders/historian/grader-support.d.mts +57 -0
- package/graders/historian/grader-support.mjs +137 -0
- package/graders/historian/grader.mjs +113 -0
- package/graders/historian/mutate.sh +114 -0
- package/graders/historian/reset-sandbox.sh +60 -0
- package/graders/historian/run-scenario.sh +49 -0
- package/graders/historian/seed-wrapped.sh +32 -0
- package/package.json +42 -0
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// Structured friction digests (plan todo 11a): every run appends ONE fixed-shape
|
|
2
|
+
// friction_digest record to the global queue <configDir>/friction.jsonl (todo-2
|
|
3
|
+
// guarded helpers — lock + schema envelope + tail repair come free). Stalls,
|
|
4
|
+
// repeated rejections (todo-8 stage/reason counts), inconclusive causes and CLI
|
|
5
|
+
// errors are COUNTS and ENUMS, never free text: every string field passes
|
|
6
|
+
// scrub() (printable-ASCII, single line, bounded).
|
|
7
|
+
//
|
|
8
|
+
// VAL LEAK GATE (plan 157-164, the core rule): material from val-split units
|
|
9
|
+
// contributes counts, alias positions and sample numbers ONLY — unit ids,
|
|
10
|
+
// paths, scenario text and failure strings sourced from val runs can never
|
|
11
|
+
// enter a record, because the queue is mutator-adjacent context downstream.
|
|
12
|
+
import { z } from "zod";
|
|
13
|
+
import { appendFriction, LedgerError, readFriction, } from "../ledger.js";
|
|
14
|
+
export const FRICTION_KIND = "friction_digest";
|
|
15
|
+
/** Rejection stages as produced by todo-8 (candidate.ts schema|syntax|path + driver apply|parse). */
|
|
16
|
+
export const REJECT_STAGES = ["schema", "syntax", "path", "apply", "parse"];
|
|
17
|
+
const MAX_REASONS = 10;
|
|
18
|
+
const MAX_TRAIN_UNITS = 32;
|
|
19
|
+
const MAX_FAILURES_PER_UNIT = 4;
|
|
20
|
+
const MAX_REASON_CHARS = 200;
|
|
21
|
+
const MAX_UNIT_ID_CHARS = 120;
|
|
22
|
+
/** Printable-ASCII, single line, bounded — the queue never carries ANSI or newlines. */
|
|
23
|
+
export function scrub(value, max = MAX_REASON_CHARS) {
|
|
24
|
+
const one = value.replace(/[^ -~]/g, " ").replace(/\s+/g, " ").trim();
|
|
25
|
+
if (one.length <= max)
|
|
26
|
+
return one;
|
|
27
|
+
return `${one.slice(0, Math.max(0, max - 3))}...`;
|
|
28
|
+
}
|
|
29
|
+
export const frictionCauseSchema = z.enum(["run-summary", "cli-error", "self-eval"]);
|
|
30
|
+
export const frictionStageCountsSchema = z.strictObject({
|
|
31
|
+
schema: z.number().int().nonnegative(),
|
|
32
|
+
syntax: z.number().int().nonnegative(),
|
|
33
|
+
path: z.number().int().nonnegative(),
|
|
34
|
+
apply: z.number().int().nonnegative(),
|
|
35
|
+
parse: z.number().int().nonnegative(),
|
|
36
|
+
});
|
|
37
|
+
export const frictionDigestSchema = z.strictObject({
|
|
38
|
+
genomeFp: z.string().regex(/^[0-9a-f]{16}$/),
|
|
39
|
+
cause: frictionCauseSchema,
|
|
40
|
+
exit: z.union([z.literal(0), z.literal(1), z.literal(2)]),
|
|
41
|
+
complete: z.boolean(),
|
|
42
|
+
counts: z.strictObject({
|
|
43
|
+
applied: z.number().int().nonnegative(),
|
|
44
|
+
rejected: z.number().int().nonnegative(),
|
|
45
|
+
benched: z.number().int().nonnegative(),
|
|
46
|
+
inconclusive: z.number().int().nonnegative(),
|
|
47
|
+
nominated: z.number().int().nonnegative(),
|
|
48
|
+
timeouts: z.number().int().nonnegative(),
|
|
49
|
+
reaped: z.number().int().nonnegative(),
|
|
50
|
+
}),
|
|
51
|
+
rejections: z.strictObject({ total: z.number().int().nonnegative(), byStage: frictionStageCountsSchema }),
|
|
52
|
+
stall: z.strictObject({ budgetTruncated: z.boolean(), orphanGroups: z.number().int().nonnegative() }),
|
|
53
|
+
train: z
|
|
54
|
+
.array(z.strictObject({
|
|
55
|
+
unitId: z.string().min(1).max(MAX_UNIT_ID_CHARS),
|
|
56
|
+
n: z.number().int().nonnegative(),
|
|
57
|
+
mean: z.number().finite(),
|
|
58
|
+
failures: z.array(z.string().min(1).max(MAX_REASON_CHARS)).max(MAX_FAILURES_PER_UNIT),
|
|
59
|
+
}))
|
|
60
|
+
.max(MAX_TRAIN_UNITS),
|
|
61
|
+
val: z.strictObject({
|
|
62
|
+
count: z.number().int().nonnegative(),
|
|
63
|
+
aliases: z.array(z.string().regex(/^val-[0-9]+$/)).max(64),
|
|
64
|
+
samples: z.number().int().nonnegative(),
|
|
65
|
+
}),
|
|
66
|
+
reasons: z.array(z.string().min(1).max(MAX_REASON_CHARS)).max(MAX_REASONS),
|
|
67
|
+
});
|
|
68
|
+
function meanOf(scores) {
|
|
69
|
+
if (scores.length === 0)
|
|
70
|
+
return 0;
|
|
71
|
+
return scores.reduce((a, b) => a + b, 0) / scores.length;
|
|
72
|
+
}
|
|
73
|
+
/** Pure builder: sanitizes + bounds every string, then schema-parses (fail-closed). */
|
|
74
|
+
export function buildRunFriction(input) {
|
|
75
|
+
const byStage = { schema: 0, syntax: 0, path: 0, apply: 0, parse: 0 };
|
|
76
|
+
const unknownStages = [];
|
|
77
|
+
for (const rejection of input.rejected) {
|
|
78
|
+
const hit = REJECT_STAGES.find((stage) => stage === rejection.stage);
|
|
79
|
+
if (hit !== undefined)
|
|
80
|
+
byStage[hit] += 1;
|
|
81
|
+
else
|
|
82
|
+
unknownStages.push(`rejection stage '${scrub(rejection.stage, 40)}': ${scrub(rejection.reason)}`);
|
|
83
|
+
}
|
|
84
|
+
const train = [];
|
|
85
|
+
let valCount = 0;
|
|
86
|
+
let valSamples = 0;
|
|
87
|
+
const valAliases = [];
|
|
88
|
+
for (const unit of input.units) {
|
|
89
|
+
if (unit.split === "val") {
|
|
90
|
+
valAliases.push(`val-${String(valCount + 1)}`);
|
|
91
|
+
valCount += 1;
|
|
92
|
+
valSamples += unit.scores.length;
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
if (train.length >= MAX_TRAIN_UNITS)
|
|
96
|
+
continue;
|
|
97
|
+
train.push({
|
|
98
|
+
unitId: scrub(unit.unitId, MAX_UNIT_ID_CHARS),
|
|
99
|
+
n: unit.scores.length,
|
|
100
|
+
mean: Number(meanOf(unit.scores).toFixed(6)),
|
|
101
|
+
failures: unit.failures.map((f) => scrub(f)).filter((f) => f.length > 0).slice(0, MAX_FAILURES_PER_UNIT),
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
const data = frictionDigestSchema.parse({
|
|
105
|
+
genomeFp: input.genomeFp,
|
|
106
|
+
cause: input.cause,
|
|
107
|
+
exit: input.exit,
|
|
108
|
+
complete: input.complete,
|
|
109
|
+
counts: input.counts,
|
|
110
|
+
rejections: { total: input.rejected.length, byStage },
|
|
111
|
+
stall: { budgetTruncated: input.stall.budgetTruncated, orphanGroups: input.stall.orphanGroups },
|
|
112
|
+
train,
|
|
113
|
+
val: { count: valCount, aliases: valAliases, samples: valSamples },
|
|
114
|
+
reasons: [...input.reasons.map((r) => scrub(r)).filter((r) => r.length > 0).slice(0, MAX_REASONS), ...unknownStages].slice(0, MAX_REASONS),
|
|
115
|
+
});
|
|
116
|
+
return input.runId === undefined
|
|
117
|
+
? { kind: FRICTION_KIND, data }
|
|
118
|
+
: { kind: FRICTION_KIND, runId: scrub(input.runId, 160), data };
|
|
119
|
+
}
|
|
120
|
+
export function appendRunFriction(configDir, input, opts = {}) {
|
|
121
|
+
return appendFriction(configDir, buildRunFriction(input), opts);
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Fail-closed queue reader: a corrupt line yields a readable error naming the
|
|
125
|
+
* file and line (never a crash, never a silent skip); foreign record kinds are
|
|
126
|
+
* refused like corruption — the queue holds friction_digest records only.
|
|
127
|
+
*/
|
|
128
|
+
export function readFrictionDigests(configDir) {
|
|
129
|
+
let records;
|
|
130
|
+
try {
|
|
131
|
+
records = readFriction(configDir);
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
if (error instanceof LedgerError)
|
|
135
|
+
return { digests: [], error: error.message };
|
|
136
|
+
throw error;
|
|
137
|
+
}
|
|
138
|
+
const digests = [];
|
|
139
|
+
for (const [index, record] of records.entries()) {
|
|
140
|
+
if (record.kind !== FRICTION_KIND) {
|
|
141
|
+
return { digests, error: `friction: line ${String(index + 1)} has kind '${record.kind}', expected '${FRICTION_KIND}'` };
|
|
142
|
+
}
|
|
143
|
+
const parsed = frictionDigestSchema.safeParse(record.data);
|
|
144
|
+
if (!parsed.success) {
|
|
145
|
+
return { digests, error: `friction: line ${String(index + 1)} failed ${FRICTION_KIND} schema: ${parsed.error.issues[0]?.message ?? "schema"}` };
|
|
146
|
+
}
|
|
147
|
+
digests.push(parsed.data);
|
|
148
|
+
}
|
|
149
|
+
return { digests, error: null };
|
|
150
|
+
}
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
// Reflection-driven mutator session driver (todo 8, plan lines 133-140).
|
|
2
|
+
//
|
|
3
|
+
// runMutatorSession launches the candidate generator as a runCommand-template child
|
|
4
|
+
// (opencode or a stub script) whose --dir points at a throwaway worktree snapshotted
|
|
5
|
+
// from the incumbent head (todo-3 store). The child's stdout must be
|
|
6
|
+
// { candidates: [{ id?, rationale, diffs: [unified diff strings] } ] } (zod-checked).
|
|
7
|
+
// Each candidate is validated (candidate.ts) and applied ONLY inside its own throwaway
|
|
8
|
+
// worktree — the real repo worktree is never touched — then sealed (a real commit) or
|
|
9
|
+
// discarded. EVERY rejection (schema/syntax/path/apply/parse) is booked in the genome
|
|
10
|
+
// ledger as kind "candidate_rejected". A missing mutator binary exits 2 BEFORE spawning.
|
|
11
|
+
//
|
|
12
|
+
// The brief policy lives in brief.ts. This module re-exports the whole todo-9 surface:
|
|
13
|
+
// buildBrief / runMutatorSession / validateCandidate + ARTIFACT_GLOBS.
|
|
14
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
15
|
+
import * as os from "node:os";
|
|
16
|
+
import path from "node:path";
|
|
17
|
+
import { blocked, cannotAnswer } from "../../exit.js";
|
|
18
|
+
import { renderCommand, runChild } from "../../bench/adapter.js";
|
|
19
|
+
import { fingerprint, genId } from "../ids.js";
|
|
20
|
+
import { Ledger } from "../ledger.js";
|
|
21
|
+
import { fingerprint16 } from "../genome.js";
|
|
22
|
+
import { newGeneration, openGenome, sealGeneration } from "../worktree.js";
|
|
23
|
+
import { formatZodIssues } from "../spec.js";
|
|
24
|
+
import { ARTIFACT_GLOBS, artifactGlobsFromGitignore, candidateIdOf, mutatorOutputSchema, validateCandidate, } from "./candidate.js";
|
|
25
|
+
import { applyChanges } from "./udiff.js";
|
|
26
|
+
export { buildBrief } from "./brief.js";
|
|
27
|
+
export { ARTIFACT_GLOBS, artifactGlobsFromGitignore, candidateSchema, mutatorOutputSchema, validateCandidate, } from "./candidate.js";
|
|
28
|
+
export { parseUnifiedDiff, applyChanges } from "./udiff.js";
|
|
29
|
+
function isExecutableFile(p) {
|
|
30
|
+
try {
|
|
31
|
+
const st = statSync(p);
|
|
32
|
+
return st.isFile() && (st.mode & 0o111) !== 0;
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/** Fail-closed bin resolution BEFORE any spawn (no raw ENOENT, plan: exit 2). */
|
|
39
|
+
function resolveBin(name, opencodeBin) {
|
|
40
|
+
if (name === "opencode" && opencodeBin !== null) {
|
|
41
|
+
if (isExecutableFile(opencodeBin))
|
|
42
|
+
return opencodeBin;
|
|
43
|
+
cannotAnswer(`mutator: configured opencodeBin is not executable: ${opencodeBin}`, "set opencodeBin in the abathur config to a working opencode binary");
|
|
44
|
+
}
|
|
45
|
+
if (name.includes("/")) {
|
|
46
|
+
const resolved = path.resolve(name);
|
|
47
|
+
if (isExecutableFile(resolved))
|
|
48
|
+
return resolved;
|
|
49
|
+
cannotAnswer(`mutator: command binary not found or not executable: ${name}`);
|
|
50
|
+
}
|
|
51
|
+
for (const dir of (process.env["PATH"] ?? "").split(path.delimiter)) {
|
|
52
|
+
if (dir === "")
|
|
53
|
+
continue;
|
|
54
|
+
const candidate = path.join(dir, name);
|
|
55
|
+
if (isExecutableFile(candidate))
|
|
56
|
+
return candidate;
|
|
57
|
+
}
|
|
58
|
+
const detail = name === "opencode" ? " (opencodeBin unset and opencode not on PATH)" : "";
|
|
59
|
+
cannotAnswer(`mutator: binary '${name}' not found${detail}`, "install the mutator CLI or set opencodeBin in the config");
|
|
60
|
+
}
|
|
61
|
+
function oneLine(text, max) {
|
|
62
|
+
const flat = text.replace(/[\r\n\t]+/g, " ");
|
|
63
|
+
return flat.length > max ? `${flat.slice(0, max)}…` : flat;
|
|
64
|
+
}
|
|
65
|
+
function sealMessage(candidateId, rationale) {
|
|
66
|
+
const safe = rationale.replace(/[^ -~]/g, " ");
|
|
67
|
+
return `abathur: mutator candidate ${candidateId} — ${oneLine(safe, 80)}`;
|
|
68
|
+
}
|
|
69
|
+
function readIfText(filePath) {
|
|
70
|
+
return existsSync(filePath) ? readFileSync(filePath, "utf8") : null;
|
|
71
|
+
}
|
|
72
|
+
/** Parse the child's stdout contract; a broken batch is a clean fatal error AFTER ledger booking. */
|
|
73
|
+
function parseStdout(stdout, reject) {
|
|
74
|
+
let doc;
|
|
75
|
+
try {
|
|
76
|
+
doc = JSON.parse(stdout);
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
const preview = oneLine(stdout, 200);
|
|
80
|
+
reject("mutator-stdout", "parse", `stdout is not JSON: ${preview}`);
|
|
81
|
+
blocked(`mutator: stdout is not JSON — cannot use this candidate batch (preview: ${preview})`);
|
|
82
|
+
}
|
|
83
|
+
const parsed = mutatorOutputSchema.safeParse(doc);
|
|
84
|
+
if (!parsed.success) {
|
|
85
|
+
const reason = `expected an object with a non-empty 'candidates' array: ${formatZodIssues(parsed.error.issues).join("; ")}`;
|
|
86
|
+
reject("mutator-stdout", "schema", reason);
|
|
87
|
+
blocked(`mutator: output schema rejected — ${reason}`);
|
|
88
|
+
}
|
|
89
|
+
return parsed.data;
|
|
90
|
+
}
|
|
91
|
+
export async function runMutatorSession(opts) {
|
|
92
|
+
const { spec } = opts;
|
|
93
|
+
const env = opts.env ?? process.env;
|
|
94
|
+
const now = opts.now ?? (() => new Date());
|
|
95
|
+
const timeoutS = opts.timeoutS ?? spec.bench.timeoutS;
|
|
96
|
+
const maxCandidates = opts.maxCandidates ?? spec.budget.maxCandidates;
|
|
97
|
+
const probeArgv = renderCommand(opts.mutatorCommand, { worktree: "", brief: "" });
|
|
98
|
+
const bin = resolveBin(probeArgv[0] ?? "", opts.opencodeBin ?? null);
|
|
99
|
+
const opened = await openGenome(spec.repoPath, [], { env });
|
|
100
|
+
const genome = { repoPath: spec.repoPath, genomeFp: fingerprint16(spec) };
|
|
101
|
+
const ledger = opts.ledger ?? Ledger.open(spec.repoPath);
|
|
102
|
+
const gitignoreText = readIfText(path.join(spec.repoPath, ".gitignore"));
|
|
103
|
+
const artifactGlobs = gitignoreText === null ? ARTIFACT_GLOBS : [...ARTIFACT_GLOBS, ...artifactGlobsFromGitignore(gitignoreText)];
|
|
104
|
+
const policy = { immutableGlobs: spec.kernel.immutableGlobs, artifactGlobs };
|
|
105
|
+
const briefDir = mkdtempSync(path.join(os.tmpdir(), "abathur-brief-"));
|
|
106
|
+
const briefFile = path.join(briefDir, "brief.md");
|
|
107
|
+
writeFileSync(briefFile, opts.brief, "utf8");
|
|
108
|
+
const applied = [];
|
|
109
|
+
const rejected = [];
|
|
110
|
+
const reject = (candidateId, stage, reason, genIdValue) => {
|
|
111
|
+
rejected.push({ candidateId, stage, reason });
|
|
112
|
+
ledger.append({
|
|
113
|
+
kind: "candidate_rejected",
|
|
114
|
+
...(genIdValue === undefined ? {} : { genId: genIdValue }),
|
|
115
|
+
data: { candidateId, stage, reason },
|
|
116
|
+
});
|
|
117
|
+
};
|
|
118
|
+
let launchGenId = "";
|
|
119
|
+
try {
|
|
120
|
+
launchGenId = genId(fingerprint({ abathur: "mutator-launch", head: opened.headCommit, at: now().getTime() }), now());
|
|
121
|
+
const launch = await newGeneration(genome, opened.headCommit, launchGenId, { env });
|
|
122
|
+
try {
|
|
123
|
+
const argv = renderCommand(opts.mutatorCommand, { worktree: launch.worktreePath, brief: briefFile });
|
|
124
|
+
const child = await runChild({
|
|
125
|
+
argv: [bin, ...argv.slice(1)],
|
|
126
|
+
cwd: launch.worktreePath,
|
|
127
|
+
timeoutS,
|
|
128
|
+
...(opts.onChild === undefined ? {} : { onChild: opts.onChild }),
|
|
129
|
+
});
|
|
130
|
+
if (child.kind !== "exited") {
|
|
131
|
+
blocked(`mutator: child ${child.kind}: ${child.reason}`);
|
|
132
|
+
}
|
|
133
|
+
const doc = parseStdout(child.stdout, reject);
|
|
134
|
+
const raws = doc.candidates.slice(0, maxCandidates);
|
|
135
|
+
for (const [index, raw] of raws.entries()) {
|
|
136
|
+
const id = candidateIdOf(raw, index);
|
|
137
|
+
const v = validateCandidate(raw, policy, id);
|
|
138
|
+
if (!v.ok) {
|
|
139
|
+
reject(id, v.stage, v.reason);
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
const gen = genId(fingerprint({ parent: opened.headCommit, candidateId: v.candidateId, changes: v.changes, at: now().getTime(), index }), now());
|
|
143
|
+
const wt = await newGeneration(genome, opened.headCommit, gen, { env });
|
|
144
|
+
let sealedOk = false;
|
|
145
|
+
try {
|
|
146
|
+
const changes = applyChanges(v.changes, (rel) => readIfText(path.join(wt.worktreePath, rel)));
|
|
147
|
+
if (!changes.ok) {
|
|
148
|
+
reject(v.candidateId, "apply", changes.reason, gen);
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
let wrote = 0;
|
|
152
|
+
for (const [rel, content] of changes.files) {
|
|
153
|
+
if (content === readIfText(path.join(wt.worktreePath, rel)))
|
|
154
|
+
continue;
|
|
155
|
+
const target = path.join(wt.worktreePath, rel);
|
|
156
|
+
mkdirSync(path.dirname(target), { recursive: true });
|
|
157
|
+
writeFileSync(target, content, "utf8");
|
|
158
|
+
wrote += 1;
|
|
159
|
+
}
|
|
160
|
+
if (wrote === 0) {
|
|
161
|
+
reject(v.candidateId, "apply", "candidate changes nothing (incumbent content already matches)", gen);
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
const sealed = await sealGeneration(genome, gen, sealMessage(v.candidateId, v.rationale), { env });
|
|
165
|
+
sealedOk = true;
|
|
166
|
+
applied.push({
|
|
167
|
+
candidateId: v.candidateId,
|
|
168
|
+
rationale: v.rationale,
|
|
169
|
+
genId: gen,
|
|
170
|
+
worktreePath: wt.worktreePath,
|
|
171
|
+
parentCommit: opened.headCommit,
|
|
172
|
+
commitSha: sealed.commitSha,
|
|
173
|
+
treeSha: sealed.treeSha,
|
|
174
|
+
touchedFiles: changes.touched,
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
finally {
|
|
178
|
+
if (!sealedOk)
|
|
179
|
+
rmSync(wt.worktreePath, { recursive: true, force: true });
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
finally {
|
|
184
|
+
rmSync(launch.worktreePath, { recursive: true, force: true });
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
finally {
|
|
188
|
+
rmSync(briefDir, { recursive: true, force: true });
|
|
189
|
+
}
|
|
190
|
+
return { launchGenId, applied, rejected };
|
|
191
|
+
}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
// Budgeted bench execution for the run-loop (todo 9): adapter factory +
|
|
2
|
+
// reset→seed→run→score matrix driver. Row schemas / resume readers: run-rows.ts.
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { cannotAnswer } from "../../exit.js";
|
|
5
|
+
import { runId } from "../ids.js";
|
|
6
|
+
import { buildManifest } from "../kernel.js";
|
|
7
|
+
import { budgetExhausted } from "../stats.js";
|
|
8
|
+
import { ToyBenchAdapter } from "../../bench/toy.js";
|
|
9
|
+
import { FixtureScenariosAdapter } from "../../bench/fixture.js";
|
|
10
|
+
import { round3, unitMatrixRowSchema } from "./run-rows.js";
|
|
11
|
+
import { selfBench } from "./self-snapshot.js";
|
|
12
|
+
export function cloneMatrixRow(u) {
|
|
13
|
+
return { unitId: u.unitId, split: u.split, scores: [...u.scores], runIds: [...u.runIds], failures: [...u.failures] };
|
|
14
|
+
}
|
|
15
|
+
export function copyProvenance(p) {
|
|
16
|
+
return { benchType: p.benchType, versions: p.versions.map((v) => ({ bin: v.bin, version: v.version })) };
|
|
17
|
+
}
|
|
18
|
+
// Unscored units (run timeout/infra_failed, inconclusive grader, budget cut) must be
|
|
19
|
+
// excluded before stats.evaluate — empty replicate lists poison aggregateScore with NaN.
|
|
20
|
+
export function asReplicates(rows) {
|
|
21
|
+
return rows.filter((u) => u.scores.length > 0).map((u) => ({ unitId: u.unitId, split: u.split, scores: u.scores }));
|
|
22
|
+
}
|
|
23
|
+
export * from "./run-rows.js";
|
|
24
|
+
/** toy ⇒ stateless adapter; fixture ⇒ ctor acquires the fingerprint lock immediately. */
|
|
25
|
+
export function openBenchAdapter(spec, opts) {
|
|
26
|
+
const childOpts = opts.onChild === undefined ? {} : { onChild: opts.onChild };
|
|
27
|
+
const envOpts = opts.env === undefined ? {} : { env: opts.env };
|
|
28
|
+
const binOpts = opts.opencodeBin === undefined ? {} : { opencodeBin: opts.opencodeBin };
|
|
29
|
+
switch (spec.bench.type) {
|
|
30
|
+
case "toy":
|
|
31
|
+
return { adapter: new ToyBenchAdapter(spec, { ...childOpts }), release: () => { } };
|
|
32
|
+
case "opencode-fixture-scenarios": {
|
|
33
|
+
const adapter = new FixtureScenariosAdapter(spec, {
|
|
34
|
+
configDir: opts.configDir,
|
|
35
|
+
// plan line 118: val paths surface in the manifest ONLY when the
|
|
36
|
+
// operator passed --include-val; authority to BENCH them is separate.
|
|
37
|
+
includeVal: opts.includeVal === true,
|
|
38
|
+
loopValAuthority: opts.loopValAuthority === true,
|
|
39
|
+
...envOpts,
|
|
40
|
+
...binOpts,
|
|
41
|
+
...childOpts,
|
|
42
|
+
});
|
|
43
|
+
return { adapter, release: () => adapter.release() };
|
|
44
|
+
}
|
|
45
|
+
default: {
|
|
46
|
+
const exhaust = spec.bench.type;
|
|
47
|
+
return cannotAnswer(`bench: unsupported bench type '${String(exhaust)}'`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
// ------------------------------------------------------------- bench runner
|
|
52
|
+
/**
|
|
53
|
+
* benchTarget is the evolution loop's bench driver, and the selection gate is
|
|
54
|
+
* its consumer: nomination REQUIRES val replicates (plan lines 125-132) and
|
|
55
|
+
* SC4 benches the shipped fixture genome train+val (plan line 214) — so the loop
|
|
56
|
+
* ALWAYS benches val units, with or without the operator flag. Commit 3c6bbf9
|
|
57
|
+
* conflated this with `--include-val` and every default `abathur run` on a
|
|
58
|
+
* val-bearing fixture genome crashed (exit 2) at the first val unit, mid-bench,
|
|
59
|
+
* before any generation row; src/test/fixture-loop.test.ts pins the fix.
|
|
60
|
+
* Exposure of val ids/paths (adapter manifest) remains the operator's
|
|
61
|
+
* includeVal alone. Do NOT gate loop benching on the operator flag again.
|
|
62
|
+
*/
|
|
63
|
+
const LOOP_VAL_AUTHORITY = true;
|
|
64
|
+
/**
|
|
65
|
+
* Drive reset→seed→run→score over EVERY unit (train AND val: the nomination gate
|
|
66
|
+
* needs val replicates) `reps` times, checking the budget before each replicate.
|
|
67
|
+
* A budget stop ends the pass with complete:false; timeout/infra_failed runs and
|
|
68
|
+
* inconclusive graders are recorded as failure notes and carry NO score sample
|
|
69
|
+
* (excluded from distributions per the hr discipline, never zeroed).
|
|
70
|
+
*/
|
|
71
|
+
export async function benchTarget(o) {
|
|
72
|
+
if (o.selfBench !== undefined) {
|
|
73
|
+
const res = await selfBench({
|
|
74
|
+
spec: o.spec,
|
|
75
|
+
genomeRepo: o.selfBench.genomeRepo,
|
|
76
|
+
genomeFp: o.selfBench.genomeFp,
|
|
77
|
+
incumbentCommit: o.selfBench.incumbentCommit,
|
|
78
|
+
candidateCommit: o.selfBench.candidateCommit,
|
|
79
|
+
genId: o.genId,
|
|
80
|
+
reps: o.reps,
|
|
81
|
+
env: o.selfBench.env,
|
|
82
|
+
onChild: o.tracker.onChild,
|
|
83
|
+
});
|
|
84
|
+
return {
|
|
85
|
+
units: res.units,
|
|
86
|
+
spent: res.spent,
|
|
87
|
+
provenance: res.provenance,
|
|
88
|
+
complete: res.complete,
|
|
89
|
+
manifest: buildManifest(o.selfBench.genomeRepo, o.spec.kernel.immutableGlobs),
|
|
90
|
+
failures: res.failures,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
const rows = new Map();
|
|
94
|
+
for (const unit of o.spec.bench.units) {
|
|
95
|
+
rows.set(unit.id, { unitId: unit.id, split: unit.split, scores: [], runIds: [], failures: [] });
|
|
96
|
+
}
|
|
97
|
+
let tokens = 0;
|
|
98
|
+
let wallS = 0;
|
|
99
|
+
let provenance = null;
|
|
100
|
+
let complete = true;
|
|
101
|
+
const bundle = openBenchAdapter(o.spec, {
|
|
102
|
+
configDir: o.configDir,
|
|
103
|
+
loopValAuthority: LOOP_VAL_AUTHORITY,
|
|
104
|
+
...(o.includeVal === true ? { includeVal: true } : {}),
|
|
105
|
+
...(o.env === undefined ? {} : { env: o.env }),
|
|
106
|
+
...(o.opencodeBin === undefined ? {} : { opencodeBin: o.opencodeBin }),
|
|
107
|
+
onChild: o.tracker.onChild,
|
|
108
|
+
});
|
|
109
|
+
try {
|
|
110
|
+
o.tracker.phase(o.genId, `${o.source}-bench`);
|
|
111
|
+
outer: for (const unit of o.spec.bench.units) {
|
|
112
|
+
for (let rep = 0; rep < o.reps; rep += 1) {
|
|
113
|
+
const running = {
|
|
114
|
+
candidates: o.countersBefore.candidates,
|
|
115
|
+
modelCalls: o.countersBefore.modelCalls,
|
|
116
|
+
tokens: o.countersBefore.tokens + tokens,
|
|
117
|
+
wallS: round3(o.countersBefore.wallS + wallS),
|
|
118
|
+
};
|
|
119
|
+
if (budgetExhausted(running, o.caps)) {
|
|
120
|
+
complete = false;
|
|
121
|
+
break outer;
|
|
122
|
+
}
|
|
123
|
+
const sandbox = path.join(o.sandboxRoot, `${unit.id}-${String(rep)}`);
|
|
124
|
+
await bundle.adapter.reset(sandbox);
|
|
125
|
+
await bundle.adapter.seed(sandbox);
|
|
126
|
+
const started = Date.now();
|
|
127
|
+
const run = await bundle.adapter.run(unit, sandbox, o.spec.bench.timeoutS);
|
|
128
|
+
wallS += (Date.now() - started) / 1000;
|
|
129
|
+
provenance ??= run.benchProvenance;
|
|
130
|
+
tokens += run.metrics.tokensEst;
|
|
131
|
+
const row = rows.get(unit.id);
|
|
132
|
+
if (row === undefined)
|
|
133
|
+
cannotAnswer(`bench: unknown unit '${unit.id}' returned from spec`);
|
|
134
|
+
if (run.status !== "ok") {
|
|
135
|
+
row.failures.push(`${unit.id} rep ${String(rep)}: run ${run.status}: ${run.note ?? run.status}`);
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
const score = await bundle.adapter.score(unit);
|
|
139
|
+
if (score.kind === "scored") {
|
|
140
|
+
row.scores.push(score.result.score);
|
|
141
|
+
row.runIds.push(runId(o.genId, unit.id, rep));
|
|
142
|
+
if (!score.result.pass) {
|
|
143
|
+
row.failures.push(`${unit.id} rep ${String(rep)}: scored ${String(score.result.score)} (not passing)`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
row.failures.push(`${unit.id} rep ${String(rep)}: grader inconclusive: ${score.reason}`);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
finally {
|
|
153
|
+
bundle.release();
|
|
154
|
+
}
|
|
155
|
+
const spent = {
|
|
156
|
+
candidates: o.source === "candidate" ? 1 : 0,
|
|
157
|
+
modelCalls: 0,
|
|
158
|
+
tokens,
|
|
159
|
+
wallS: round3(wallS),
|
|
160
|
+
};
|
|
161
|
+
const failures = [...rows.values()].flatMap((r) => r.failures);
|
|
162
|
+
return {
|
|
163
|
+
units: [...rows.values()].map((r) => unitMatrixRowSchema.parse(r)),
|
|
164
|
+
spent,
|
|
165
|
+
provenance: provenance ?? { benchType: o.spec.bench.type, versions: [] },
|
|
166
|
+
complete,
|
|
167
|
+
manifest: buildManifest(path.resolve(o.spec.repoPath), o.spec.kernel.immutableGlobs),
|
|
168
|
+
failures,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// Friction aggregation for the run-loop (plan todo 11a): accumulates bench,
|
|
2
|
+
// session and verdict facts as they happen and emits ONE RunFrictionInput at
|
|
3
|
+
// run end. Lives outside run-loop.ts so the orchestrator stays focused; the
|
|
4
|
+
// loop only calls into this when the CLI injected a sink. Every string that
|
|
5
|
+
// reaches the queue passes through friction.ts buildRunFriction (scrub +
|
|
6
|
+
// bounds + val aliasing) — nothing here may add raw child output.
|
|
7
|
+
export function startRunFriction(sink) {
|
|
8
|
+
const units = new Map();
|
|
9
|
+
let rejected = [];
|
|
10
|
+
let applied = 0;
|
|
11
|
+
let benched = 0;
|
|
12
|
+
let truncated = false;
|
|
13
|
+
const reasons = [];
|
|
14
|
+
const fold = (rows) => {
|
|
15
|
+
for (const r of rows) {
|
|
16
|
+
const agg = units.get(r.unitId) ?? { unitId: r.unitId, split: r.split, scores: [], failures: [] };
|
|
17
|
+
agg.scores.push(...r.scores);
|
|
18
|
+
agg.failures.push(...r.failures);
|
|
19
|
+
units.set(r.unitId, agg);
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
return {
|
|
23
|
+
noteUnits: fold,
|
|
24
|
+
noteBench(out) {
|
|
25
|
+
benched += 1;
|
|
26
|
+
truncated ||= !out.complete;
|
|
27
|
+
fold(out.units);
|
|
28
|
+
},
|
|
29
|
+
noteSession(rows, appliedCount) {
|
|
30
|
+
rejected = rows.map((r) => ({ stage: r.stage, reason: r.reason }));
|
|
31
|
+
applied = appliedCount;
|
|
32
|
+
},
|
|
33
|
+
noteCandidate(candidateId, failures, guardNote) {
|
|
34
|
+
if (guardNote !== null)
|
|
35
|
+
reasons.push(`candidate ${candidateId}: self-bench guard ${guardNote}`);
|
|
36
|
+
for (const f of failures)
|
|
37
|
+
reasons.push(`candidate ${candidateId}: ${f}`);
|
|
38
|
+
},
|
|
39
|
+
emit(ctx) {
|
|
40
|
+
const timeouts = [...units.values()].reduce((n, u) => n + u.failures.filter((f) => f.includes(": run timeout") || f.includes(": run infra_failed")).length, 0);
|
|
41
|
+
sink({
|
|
42
|
+
genomeFp: ctx.genomeFp,
|
|
43
|
+
cause: "run-summary",
|
|
44
|
+
runId: ctx.runId,
|
|
45
|
+
exit: ctx.exit,
|
|
46
|
+
complete: !truncated,
|
|
47
|
+
counts: {
|
|
48
|
+
applied,
|
|
49
|
+
rejected: rejected.length,
|
|
50
|
+
benched,
|
|
51
|
+
inconclusive: ctx.considered.filter((v) => v === "inconclusive").length,
|
|
52
|
+
nominated: ctx.considered.filter((v) => v === "nominated").length,
|
|
53
|
+
timeouts,
|
|
54
|
+
reaped: ctx.orphanGroups,
|
|
55
|
+
},
|
|
56
|
+
rejected,
|
|
57
|
+
units: [...units.values()],
|
|
58
|
+
reasons,
|
|
59
|
+
stall: { budgetTruncated: truncated, orphanGroups: ctx.orphanGroups },
|
|
60
|
+
});
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|