@doidor/agentrig 0.8.0 → 0.10.0

package/dist/commands/eval-dynamic.js

@@ -0,0 +1,316 @@
+/**
+ * Dynamic eval orchestrator (P3-P4). Owns the per-scenario flow:
+ *   seed worktree -> producer conv (in worktree) -> capture diff
+ *   -> run oracle (hard axes) -> judge conv (separate, different family)
+ *   -> save record via score.mjs with both producer+judge model metadata.
+ *
+ * --n controls trial count; each trial is an independent seed+producer+oracle+judge cycle.
+ * --variant tags the records so `score.mjs compare --baseline` can compute lift.
+ *
+ * For `variant=baseline` the producer prompt explicitly tells the agent to ignore AGENTS.md/
+ * rules/skills; the same scenario otherwise runs identically. That's the harness on/off A/B.
+ */
+import { existsSync, mkdirSync, writeFileSync, readFileSync, cpSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { join, resolve } from "node:path";
+import { color, log } from "../core/logger.js";
+import { ActivityMonitor } from "../core/activity.js";
+import { getProvider } from "../agent/index.js";
+import { AgentTimeoutError } from "../agent/provider.js";
+import { listScenarios, locateScenario, loadScenario, loadOracle, seedWorktree, runOracle, oracleAxisScores, oracleAxesToFlags, captureDiff, } from "../core/scenario-runner.js";
+import { modelFamily, sameFamily } from "../core/model-family.js";
+import { buildProducerPrompt, buildJudgePrompt, SYSTEM_MESSAGE } from "../prompts/index.js";
+/** Stage the AgentRig harness (AGENTS.md + .agents/ + .agentrig/) into a worktree for the
+ *  "harness" variant so the producer agent sees the same instructions/skills/rules an
+ *  AgentRig-installed repo would have, then fold the staged files into the baseline commit
+ *  so they don't pollute the producer's diff. Skipped for the "baseline" variant. */
+function stageHarnessInto(worktree, repoRoot, variant) {
+    if (variant === "baseline")
+        return;
+    let staged = false;
+    for (const rel of ["AGENTS.md", ".agents", ".agentrig"]) {
+        const src = join(repoRoot, rel);
+        if (!existsSync(src))
+            continue;
+        cpSync(src, join(worktree, rel), { recursive: true });
+        staged = true;
+    }
+    if (!staged)
+        return;
+    // CRITICAL: fold the staged harness into the baseline commit. Without this, every staged
+    // file shows up in captureDiff() as part of the producer's change → oracle scope check
+    // erroneously sees "+5000 lines / 100 files" and the judge scores scope/maintainability=0
+    // for a producer who only edited two files. (Caught by user-reported eval run 2026-06-10.)
+    spawnSync("git", ["add", "-A"], { cwd: worktree });
+    spawnSync("git", ["commit", "-q", "--amend", "--no-edit"], { cwd: worktree });
+}
+/** Save a scenario score by shelling out to the installed score.mjs (single source of truth
+ *  for the on-disk format + validation). Returns the child exit status. */
+function saveScoreViaScript(repoRoot, type, scenarioId, axisFlags, opts) {
+    const script = join(repoRoot, ".agentrig", "eval", "score.mjs");
+    const args = [
+        script,
+        "save",
+        "--type", type,
+        "--task", scenarioId,
+        "--scenario", scenarioId,
+        "--judge", opts.judgeModel || "n/a",
+        "--variant", opts.variant,
+        "--run", opts.runId,
+        ...axisFlags.flatMap((f) => ["--axis", f]),
+    ];
+    // Producer/judge metadata is appended via env so score.mjs can record + validate without
+    // adding new positional flags. The new flags are picked up by env in score.mjs's save.
+    const env = {
+        ...process.env,
+        AGENTRIG_PRODUCER_MODEL: opts.producerModel || "",
+        AGENTRIG_JUDGE_MODEL: opts.judgeModel || "",
+        AGENTRIG_ALLOW_SAME_FAMILY: opts.allowSameFamily ? "1" : "",
+    };
+    const res = spawnSync(process.execPath, args, { encoding: "utf8", env });
+    return { status: res.status ?? 1, stdout: res.stdout ?? "", stderr: res.stderr ?? "" };
+}
+/** Read judge soft scores from <artifactsDir>/<scenario>.<trial>.judge.json (judge writes
+ *  this file via the prompt — provider-agnostic so we don't depend on tool-calling APIs). */
+function readJudgeScores(artifactsDir, scenarioId, trial) {
+    const path = join(artifactsDir, `${scenarioId}.trial${trial}.judge.json`);
+    if (!existsSync(path))
+        return null;
+    try {
+        const obj = JSON.parse(readFileSync(path, "utf8"));
+        return Array.isArray(obj?.axes) ? obj : null;
+    }
+    catch {
+        return null;
+    }
+}
+function judgeAxesToFlags(judgeOut) {
+    return judgeOut.axes.map((a) => {
+        if (a.confidence === 0)
+            return `${a.name}=na`;
+        if (a.score >= 1)
+            return `${a.name}=1.0`;
+        const code = a.code || "";
+        const ev = (a.evidence || "judge-scored").replace(/:/g, ";").slice(0, 200);
+        return `${a.name}=${a.score}:${code}:${ev}`;
+    });
+}
+/** Run one trial of one scenario: producer → oracle → judge → save. */
+async function runOneTrial(repoRoot, paths, trial, opts) {
+    const fm = loadScenario(paths);
+    const oracle = loadOracle(paths);
+    const wt = seedWorktree(paths.fixtureDir, opts.runId, `${fm.id}.trial${trial}`);
+    log.info(color.dim(`    [${fm.id} trial ${trial + 1}/${opts.trials}] worktree ${wt}`));
+    stageHarnessInto(wt, repoRoot, opts.variant);
+    // ---- 1. Producer ----
+    // The dogfood scenario skips the producer entirely — the oracle runs `agentrig init` itself.
+    const promptText = readFileSync(paths.promptMd, "utf8");
+    const isDogfood = fm.id === "agentrig-init-on-empty-repo";
+    let producerTranscript = "";
+    if (!isDogfood) {
+        const provider = getProvider();
+        const monitor = new ActivityMonitor().start();
+        let convo = null;
+        try {
+            convo = await provider.startConversation({
+                cwd: wt,
+                systemMessage: SYSTEM_MESSAGE,
+                onEvent: monitor.handle,
+                ...(opts.producerModel ? { model: opts.producerModel } : {}),
+                ...(opts.timeoutMinutes ? { maxMs: opts.timeoutMinutes * 60 * 1000 } : {}),
+            });
+            producerTranscript = await convo.send(buildProducerPrompt(promptText, opts.variant));
+        }
+        catch (err) {
+            if (err instanceof AgentTimeoutError) {
+                log.warn(`      producer timed out: ${err.message}`);
+                producerTranscript = `[producer timed out: ${err.message}]`;
+            }
+            else {
+                throw err;
+            }
+        }
+        finally {
+            // Stop the heartbeat first so a failure later (e.g. convo.end()) doesn't keep it
+            // ticking. setInterval was the silent leak that flooded the terminal pre-fix.
+            monitor.stop();
+            if (convo)
+                await convo.end().catch(() => undefined);
+        }
+    }
+    // ---- 2. Oracle ----
+    const oracleEnv = {};
+    if (isDogfood)
+        oracleEnv.AGENTRIG_CLI = resolve(repoRoot, "dist/cli.js");
+    const oracleResults = runOracle(wt, oracle, oracleEnv);
+    const oracleAxes = oracleAxisScores(oracleResults);
+    const oracleFlags = oracleAxesToFlags(oracleAxes, fm.type);
+    const oraclePassed = oracleAxes.every((a) => a.score === 1);
+    // Persist diff + transcript so regressions are inspectable.
+    const diff = captureDiff(wt);
+    writeFileSync(join(opts.artifactsDir, `${fm.id}.trial${trial}.diff.patch`), diff);
+    writeFileSync(join(opts.artifactsDir, `${fm.id}.trial${trial}.transcript.md`), producerTranscript || "(no producer transcript — dogfood scenario)");
+    writeFileSync(join(opts.artifactsDir, `${fm.id}.trial${trial}.oracle.json`), JSON.stringify({ axes: oracleAxes, raw: oracleResults }, null, 2));
+    // ---- 3. Judge ----
+    let judgeFlags = [];
+    const wantJudge = Array.isArray(fm.judge_axes) && fm.judge_axes.length > 0;
+    if (wantJudge) {
+        if (opts.judgeModel) {
+            const provider = getProvider();
+            const monitor = new ActivityMonitor().start();
+            const judgeOutPath = join(opts.artifactsDir, `${fm.id}.trial${trial}.judge.json`);
+            const judgeCwd = join(opts.artifactsDir, `judge.${fm.id}.trial${trial}`);
+            let convo = null;
+            let judgeFailed = false;
+            try {
+                mkdirSync(judgeCwd, { recursive: true });
+                // The judge sees ONLY: prompt, diff, transcript, oracle results, brief. NOT the producer worktree.
+                writeFileSync(join(judgeCwd, "prompt.md"), promptText);
+                writeFileSync(join(judgeCwd, "diff.patch"), diff);
+                writeFileSync(join(judgeCwd, "transcript.md"), producerTranscript || "");
+                writeFileSync(join(judgeCwd, "oracle.json"), JSON.stringify(oracleAxes, null, 2));
+                if (paths.judgeBriefMd)
+                    writeFileSync(join(judgeCwd, "judge_brief.md"), readFileSync(paths.judgeBriefMd, "utf8"));
+                convo = await provider.startConversation({
+                    cwd: judgeCwd,
+                    systemMessage: SYSTEM_MESSAGE,
+                    onEvent: monitor.handle,
+                    ...(opts.judgeModel ? { model: opts.judgeModel } : {}),
+                    ...(opts.timeoutMinutes ? { maxMs: Math.max(1, Math.floor((opts.timeoutMinutes ?? 30) / 3)) * 60 * 1000 } : {}),
+                });
+                await convo.send(buildJudgePrompt({
+                    scenario: fm.id,
+                    type: fm.type,
+                    judgeAxes: fm.judge_axes ?? [],
+                    outputJsonPath: judgeOutPath,
+                    rubricPath: resolve(repoRoot, ".agentrig/eval/axes.json"),
+                }));
+            }
+            catch (err) {
+                judgeFailed = true;
+                if (err instanceof AgentTimeoutError) {
+                    log.warn(`      judge timed out: ${err.message}`);
+                }
+                else {
+                    // Don't bail the whole trial — soft axes will just be marked na below. Log so the
+                    // operator sees the cause (e.g. "Model X is not available").
+                    log.warn(`      judge failed (${err.message}); soft axes will be na for this trial`);
+                }
+            }
+            finally {
+                // Stop the heartbeat first so any later failure (e.g. convo.end()) doesn't keep it ticking.
+                monitor.stop();
+                if (convo)
+                    await convo.end().catch(() => undefined);
+            }
+            if (!judgeFailed) {
+                const judgeOut = readJudgeScores(opts.artifactsDir, fm.id, trial);
+                if (judgeOut)
+                    judgeFlags = judgeAxesToFlags(judgeOut);
+                else
+                    log.warn(`      judge did not write ${judgeOutPath} — soft axes recorded as na`);
+            }
+        }
+        // If no judge model OR judge failed to write, mark soft axes na so they're excluded from rollup.
+        if (judgeFlags.length === 0)
+            judgeFlags = (fm.judge_axes ?? []).map((a) => `${a}=na`);
+    }
+    // ---- 4. Save ----
+    const allFlags = [...oracleFlags, ...judgeFlags];
+    const saved = saveScoreViaScript(repoRoot, fm.type, fm.id, allFlags, {
+        variant: opts.variant,
+        runId: opts.runId,
+        producerModel: opts.producerModel ?? "",
+        judgeModel: opts.judgeModel ?? "",
+        allowSameFamily: Boolean(opts.allowSameFamily),
+    });
+    const notes = saved.stderr ? saved.stderr.split("\n").filter(Boolean).slice(-2).join(" | ") : "";
+    if (saved.status !== 0)
+        log.warn(`      score.mjs save failed (${saved.status}): ${notes}`);
+    return { scenario: fm.id, trialIndex: trial, oraclePassed, saveExit: saved.status, notes };
+}
+/** Run every scenario × every trial. Returns one outcome per (scenario, trial). */
+export async function runDynamicEval(repoRoot, opts) {
+    // Resolve scenario list, then filter out `bundled: true` unless the user opted in.
+    // When --scenario <id> is explicit, we always run it (the user named it, they get it).
+    let ids;
+    let bundledExcluded = [];
+    if (opts.scenario) {
+        ids = [opts.scenario];
+    }
+    else {
+        const all = listScenarios(repoRoot);
+        ids = [];
+        for (const id of all) {
+            const paths = locateScenario(repoRoot, id);
+            if (!paths)
+                continue;
+            let fm;
+            try {
+                fm = loadScenario(paths);
+            }
+            catch {
+                continue;
+            }
+            if (fm.bundled === true && !opts.includeBundled) {
+                bundledExcluded.push({ id, reason: "bundled (template; pass --include-bundled to run it)" });
+                continue;
+            }
+            ids.push(id);
+        }
+        if (bundledExcluded.length) {
+            log.info(color.dim(`  Excluded ${bundledExcluded.length} bundled scenario(s) from default run: ${bundledExcluded.map((b) => b.id).join(", ")}`));
+            log.info(color.dim("  Pass --include-bundled to run them too, or generate repo-specific ones with `agentrig eval --scaffold`."));
+        }
+        if (ids.length === 0) {
+            log.warn("No repo-specific scenarios found.");
+            log.warn("Generate some with: agentrig eval --scaffold");
+            log.warn("Or run the bundled template scenarios anyway: agentrig eval --dynamic --include-bundled");
+            return [];
+        }
+    }
+    const outcomes = [];
+    // Family enforcement up-front so we fail fast (before the first model call).
+    if (opts.producerModel && opts.judgeModel && !opts.allowSameFamily) {
+        if (sameFamily(opts.producerModel, opts.judgeModel)) {
+            throw new Error(`producer "${opts.producerModel}" and judge "${opts.judgeModel}" share family "${modelFamily(opts.producerModel)}". ` +
+                `Pass --allow-same-family to override (recorded in every result).`);
+        }
+    }
+    // Pre-flight model validation: catch typos in producer/judge model ids before we burn 30s
+    // of producer work just to crash at the judge call. This is what bit us when the default
+    // judge model was "gpt-5" (which doesn't exist; the real id is "gpt-5.4").
+    const provider = getProvider();
+    if (provider.validateModel) {
+        for (const [role, model] of [["producer", opts.producerModel], ["judge", opts.judgeModel]]) {
+            if (!model)
+                continue;
+            const v = await provider.validateModel(model);
+            if (!v.ok) {
+                throw new Error(`${role} model "${model}" not available from provider "${provider.name}". ${v.detail ?? ""} ` +
+                    `Fix by editing .agentrig/agents/${role === "producer" ? "developer" : "reviewer"}.yml ` +
+                    `or passing --${role}-model <id>.`);
+            }
+        }
+    }
+    for (const id of ids) {
+        const paths = locateScenario(repoRoot, id);
+        if (!paths) {
+            log.warn(`scenario "${id}" not found — skipping`);
+            continue;
+        }
+        log.step(`scenario ${color.bold(id)} (${opts.trials} trial${opts.trials > 1 ? "s" : ""})`);
+        for (let trial = 0; trial < opts.trials; trial++) {
+            try {
+                const outcome = await runOneTrial(repoRoot, paths, trial, opts);
+                outcomes.push(outcome);
+            }
+            catch (err) {
+                log.error(`scenario ${id} trial ${trial + 1} failed: ${err.message}`);
+                outcomes.push({ scenario: id, trialIndex: trial, oraclePassed: false, saveExit: 1, notes: err.message });
+            }
+        }
+    }
+    return outcomes;
+}
+//# sourceMappingURL=eval-dynamic.js.map

package/dist/commands/eval-dynamic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-dynamic.js","sourceRoot":"","sources":["../../src/commands/eval-dynamic.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACrF,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EACL,aAAa,EACb,cAAc,EACd,YAAY,EACZ,UAAU,EACV,YAAY,EACZ,SAAS,EACT,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,GAEZ,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AA2B5F;;;qFAGqF;AACrF,SAAS,gBAAgB,CAAC,QAAgB,EAAE,QAAgB,EAAE,OAAe;IAC3E,IAAI,OAAO,KAAK,UAAU;QAAE,OAAO;IACnC,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,KAAK,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,CAAC;QACxD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC/B,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,CAAC,MAAM;QAAE,OAAO;IACpB,yFAAyF;IACzF,uFAAuF;IACvF,0FAA0F;IAC1F,2FAA2F;IAC3F,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IACnD,SAAS,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;AAChF,CAAC;AAED;2EAC2E;AAC3E,SAAS,kBAAkB,CACzB,QAAgB,EAChB,IAAY,EACZ,UAAkB,EAClB,SAAmB,EACnB,IAA6G;IAE7G,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAChE,MAAM,IAAI,GAAG;QACX,MAAM;QACN,MAAM;QACN,QAAQ,EAAE,IAAI;QACd,QAAQ,EAAE,UAAU;QACpB,YAAY,EAAE,UAAU;QACxB,SAAS,EAAE,IAAI,CAAC,UAAU,IAAI,KAAK;QACnC,WAAW,EAAE,IAAI,CAAC,OAAO;QACzB,OAAO,EAAE,IAAI,CAAC,KAAK;QACnB,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;KAC3C,CAAC;IACF,yFAAyF;IACzF,uFAAuF;IACvF,MAAM,GAAG,GAAsB;QAC7B,GAAG,OAAO,CAAC,GAAG;QACd,uBAAuB,EAAE,IAAI,CAAC,aAAa,IAAI,EAAE;QACjD,oBAAoB,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;QAC3C,0BAA0B,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;KAC5D,CAAC;IACF,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IACzE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;AACzF,CAAC;AAED;6FAC6F;AAC7F,SAAS,eAAe,CAAC,YAAoB,EAAE,UAAkB,EAAE,KAAa;IAC9E,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,EAAE,GAAG,UAAU,SAAS,KAAK,aAAa,CAAC,CAAC;IAC1E,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QACnD,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,QAA4G;IACpI,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC7B,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC;QAC9C,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC;QACzC,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,cAAc,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC3E,OAAO,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,uEAAuE;AACvE,KAAK,UAAU,WAAW,CACxB,QAAgB,EAChB,KAAoB,EACpB,KAAa,EACb,IAAuB;IAEvB,MAAM,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,SAAS,KAAK,EAAE,CAAC,CAAC;IAChF,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,UAAU,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,cAAc,EAAE,EAAE,CAAC,CAAC,CAAC;IACvF,gBAAgB,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAE7C,wBAAwB;IACxB,6FAA6F;IAC7F,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxD,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,KAAK,6BAA6B,CAAC;IAC1D,IAAI,kBAAkB,GAAG,EAAE,CAAC;IAE5B,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC,KAAK,EAAE,CAAC;QAC9C,IAAI,KAAK,GAAkE,IAAI,CAAC;QAChF,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,QAAQ,CAAC,iBAAiB,CAAC;gBACvC,GAAG,EAAE,EAAE;gBACP,aAAa,EAAE,cAAc;gBAC7B,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5D,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,cAAc,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC3E,CAAC,CAAC;YACH,kBAAkB,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACvF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,iBAAiB,EAAE,CAAC;gBACrC,GAAG,CAAC,IAAI,CAAC,6BAA6B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;gBACrD,kBAAkB,GAAG,wBAAwB,GAAG,CAAC,OAAO,GAAG,CAAC;YAC9D,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,iFAAiF;YACjF,8EAA8E;YAC9E,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,IAAI,KAAK;gBAAE,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,sBAAsB;IACtB,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,IAAI,SAAS;QAAE,SAAS,CAAC,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACzE,MAAM,aAAa,GAAG,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,gBAAgB,CAAC,aAAa,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,iBAAiB,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;IAC3D,MAAM,YAAY,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;IAE5D,4DAA4D;IAC5D,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC;IAC7B,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,EAAE,SAAS,KAAK,aAAa,CAAC,EAAE,IAAI,CAAC,CAAC;IAClF,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,EAAE,SAAS,KAAK,gBAAgB,CAAC,EAAE,kBAAkB,IAAI,6CAA6C,CAAC,CAAC;IACpJ,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,EAAE,SAAS,KAAK,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAEhJ,qBAAqB;IACrB,IAAI,UAAU,GAAa,EAAE,CAAC;IAC9B,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;IAC3E,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC,KAAK,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,EAAE,SAAS,KAAK,aAAa,CAAC,CAAC;YAClF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,KAAK,EAAE,CAAC,CAAC;YACzE,IAAI,KAAK,GAAkE,IAAI,CAAC;YAChF,IAAI,WAAW,GAAG,KAAK,CAAC;YACxB,IAAI,CAAC;gBACH,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBACzC,mGAAmG;gBACnG,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,UAAU,CAAC,CAAC;gBACvD,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,EAAE,IAAI,CAAC,CAAC;gBAClD,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,EAAE,kBAAkB,IAAI,EAAE,CAAC,CAAC;gBACzE,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;gBAClF,IAAI,KAAK,CAAC,YAAY;oBAAE,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,EAAE,YAAY,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;gBAClH,KAAK,GAAG,MAAM,QAAQ,CAAC,iBAAiB,CAAC;oBACvC,GAAG,EAAE,QAAQ;oBACb,aAAa,EAAE,cAAc;oBAC7B,OAAO,EAAE,OAAO,CAAC,MAAM;oBACvB,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtD,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAChH,CAAC,CAAC;gBACH,MAAM,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC;oBAChC,QAAQ,EAAE,EAAE,CAAC,EAAE;oBACf,IAAI,EAAE,EAAE,CAAC,IAAI;oBACb,SAAS,EAAE,EAAE,CAAC,UAAU,IAAI,EAAE;oBAC9B,cAAc,EAAE,YAAY;oBAC5B,UAAU,EAAE,OAAO,CAAC,QAAQ,EAAE,0BAA0B,CAAC;iBAC1D,CAAC,CAAC,CAAC;YACN,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,WAAW,GAAG,IAAI,CAAC;gBACnB,IAAI,GAAG,YAAY,iBAAiB,EAAE,CAAC;oBACrC,GAAG,CAAC,IAAI,CAAC,0BAA0B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,kFAAkF;oBAClF,6DAA6D;oBAC7D,GAAG,CAAC,IAAI,CAAC,uBAAwB,GAAa,CAAC,OAAO,wCAAwC,CAAC,CAAC;gBAClG,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,4FAA4F;gBAC5F,OAAO,CAAC,IAAI,EAAE,CAAC;gBACf,IAAI,KAAK;oBAAE,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;gBAClE,IAAI,QAAQ;oBAAE,UAAU,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;;oBACjD,GAAG,CAAC,IAAI,CAAC,6BAA6B,YAAY,6BAA6B,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;QACD,iGAAiG;QACjG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,UAAU,GAAG,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACxF,CAAC;IAED,oBAAoB;IACpB,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,UAAU,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,kBAAkB,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE;QACnE,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,EAAE;QACvC,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;QACjC,eAAe,EAAE,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC;KAC/C,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACjG,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,gCAAgC,KAAK,CAAC,MAAM,MAAM,KAAK,EAAE,CAAC,CAAC;IAE5F,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;AAC7F,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB,EAAE,IAAuB;IAC5E,mFAAmF;IACnF,uFAAuF;IACvF,IAAI,GAAa,CAAC;IAClB,IAAI,eAAe,GAAqC,EAAE,CAAC;IAC3D,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxB,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QACpC,GAAG,GAAG,EAAE,CAAC;QACT,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC3C,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,IAAI,EAAE,CAAC;YACP,IAAI,CAAC;gBAAC,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC;gBAAC,SAAS;YAAC,CAAC;YACrD,IAAI,EAAE,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;gBAChD,eAAe,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,sDAAsD,EAAE,CAAC,CAAC;gBAC7F,SAAS;YACX,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,CAAC;QACD,IAAI,eAAe,CAAC,MAAM,EAAE,CAAC;YAC3B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,eAAe,CAAC,MAAM,0CAA0C,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACjJ,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,2GAA2G,CAAC,CAAC,CAAC;QACnI,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,GAAG,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;YAC9C,GAAG,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;YACzD,GAAG,CAAC,IAAI,CAAC,yFAAyF,CAAC,CAAC;YACpG,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAsB,EAAE,CAAC;IAEvC,6EAA6E;IAC7E,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;QACnE,IAAI,UAAU,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CACb,aAAa,IAAI,CAAC,aAAa,gBAAgB,IAAI,CAAC,UAAU,mBAAmB,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK;gBACrH,kEAAkE,CACnE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,0FAA0F;IAC1F,yFAAyF;IACzF,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CAAU,EAAE,CAAC;YACpG,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAC9C,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CACb,GAAG,IAAI,WAAW,KAAK,kCAAkC,QAAQ,CAAC,IAAI,MAAM,CAAC,CAAC,MAAM,IAAI,EAAE,GAAG;oBAC7F,mCAAmC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,OAAO;oBACxF,gBAAgB,IAAI,cAAc,CACnC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,wBAAwB,CAAC,CAAC;YAClD,SAAS;QACX,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,YAAY,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,MAAM,SAAS,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC3F,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;gBAChE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,KAAK,GAAG,CAAC,YAAa,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;gBACjF,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAG,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACtH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/commands/eval-scaffold.js

@@ -0,0 +1,173 @@
+/**
+ * `agentrig eval --scaffold` orchestrator. Asks the agent to generate repo-specific scenarios
+ * alongside the bundled generic ones, then validates each new scenario's structure via the
+ * existing scenario-runner load logic. No-ops for invalid output rather than corrupting the
+ * eval surface.
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { ActivityMonitor } from "../core/activity.js";
+import { color, log } from "../core/logger.js";
+import { getProvider } from "../agent/index.js";
+import { AgentTimeoutError } from "../agent/provider.js";
+import { listScenarios, locateScenario, loadScenario, loadOracle, } from "../core/scenario-runner.js";
+import { buildScaffoldScenariosPrompt, SYSTEM_MESSAGE } from "../prompts/index.js";
+import { join } from "../core/fsutil.js";
+/** Discover up to 3 existing scenarios to use as templates for the agent. Prefers
+ *  small fixtures so the prompt stays compact. */
+function collectExamples(repoRoot) {
+    const ids = listScenarios(repoRoot).filter((id) => id !== "agentrig-init-on-empty-repo");
+    const out = [];
+    for (const id of ids.slice(0, 3)) {
+        const paths = locateScenario(repoRoot, id);
+        if (!paths)
+            continue;
+        out.push({
+            id,
+            scenarioYml: existsSync(paths.scenarioYml) ? readFileSync(paths.scenarioYml, "utf8") : "",
+            promptMd: existsSync(paths.promptMd) ? readFileSync(paths.promptMd, "utf8") : "",
+            oracleYml: existsSync(paths.oracleYml) ? readFileSync(paths.oracleYml, "utf8") : "",
+        });
+    }
+    return out;
+}
+/** Pull the allowed axis names from the live axes.json registry so the prompt can constrain
+ *  the agent to existing axes (otherwise the new scenario won't score). */
+function collectAxesAvailable(repoRoot) {
+    const axesPath = join(repoRoot, ".agentrig/eval/axes.json");
+    if (!existsSync(axesPath))
+        return { types: [], axisNames: [] };
+    const axes = JSON.parse(readFileSync(axesPath, "utf8"));
+    const types = Object.keys(axes.types ?? {});
+    const axisNames = new Set();
+    for (const t of Object.values(axes.types ?? {})) {
+        for (const cat of Object.values(t.categories ?? {})) {
+            for (const axis of Object.keys(cat))
+                axisNames.add(axis);
+        }
+    }
+    return { types, axisNames: [...axisNames].sort() };
+}
+/** Validate one newly-generated scenario. Catches parse errors + obvious schema gaps
+ *  (unknown axes, missing fixture dir) before the user runs `eval --dynamic`. */
+function validateScaffoldedScenario(repoRoot, id, allowedAxes) {
+    const paths = locateScenario(repoRoot, id);
+    if (!paths)
+        return { ok: false, reason: "scenario directory disappeared after agent run" };
+    if (!existsSync(paths.fixtureDir))
+        return { ok: false, reason: "missing fixture/ directory" };
+    let fm;
+    try {
+        fm = loadScenario(paths);
+    }
+    catch (e) {
+        return { ok: false, reason: `scenario.yml parse error: ${e.message}` };
+    }
+    let oracle;
+    try {
+        oracle = loadOracle(paths);
+    }
+    catch (e) {
+        return { ok: false, reason: `oracle.yml parse error: ${e.message}` };
+    }
+    // Every axis referenced (oracle_axes, judge_axes, oracle.checks[].axis) must exist in axes.json.
+    const referenced = new Set([
+        ...(fm.oracle_axes ?? []),
+        ...(fm.judge_axes ?? []),
+        ...oracle.checks.map((c) => c.axis),
+    ]);
+    const unknown = [...referenced].filter((a) => a && !allowedAxes.has(a));
+    if (unknown.length)
+        return { ok: false, reason: `unknown axes: ${unknown.join(", ")}` };
+    if (!existsSync(paths.promptMd))
+        return { ok: false, reason: "missing prompt.md" };
+    return { ok: true };
+}
+export async function scaffoldScenarios(repoRoot, opts) {
+    const provider = getProvider();
+    const pre = await provider.preflight();
+    if (!pre.ok)
+        throw new Error(`agent unavailable: ${pre.detail}`);
+    log.ok(`agent ready (${pre.detail})`);
+    if (opts.producerModel && provider.validateModel) {
+        const v = await provider.validateModel(opts.producerModel);
+        if (!v.ok)
+            throw new Error(`model "${opts.producerModel}" not available. ${v.detail ?? ""}`);
+    }
+    const before = new Set(listScenarios(repoRoot));
+    const examples = collectExamples(repoRoot);
+    if (examples.length === 0) {
+        throw new Error("no existing scenarios found to use as templates. Run `agentrig init` first so the " +
+            "generic scenarios are installed at .agentrig/eval/scenarios/.");
+    }
+    const axesAvailable = collectAxesAvailable(repoRoot);
+    if (axesAvailable.axisNames.length === 0) {
+        throw new Error("no axes registered at .agentrig/eval/axes.json — run `agentrig init` first.");
+    }
+    const contextMdPath = join(repoRoot, ".agentrig/context.md");
+    const contextMd = existsSync(contextMdPath) ? readFileSync(contextMdPath, "utf8") : "";
+    if (!contextMd) {
+        log.warn("no .agentrig/context.md found — the agent will investigate the repo itself,");
+        log.warn("which adds time. Consider running `agentrig init` (without --skip-agent) first.");
+    }
+    log.step(`scaffolding ${opts.count} repo-specific scenario(s)…`);
+    log.info(color.dim(`  Using ${examples.length} existing scenario(s) as templates: ${examples.map((e) => e.id).join(", ")}`));
+    log.info(color.dim(`  Available axes (${axesAvailable.axisNames.length}): ${axesAvailable.axisNames.slice(0, 6).join(", ")}…`));
+    const monitor = new ActivityMonitor().start();
+    let convo = null;
+    try {
+        convo = await provider.startConversation({
+            cwd: repoRoot,
+            systemMessage: SYSTEM_MESSAGE,
+            onEvent: monitor.handle,
+            ...(opts.producerModel ? { model: opts.producerModel } : {}),
+            ...(opts.timeoutMinutes ? { maxMs: opts.timeoutMinutes * 60 * 1000 } : {}),
+        });
+        await convo.send(buildScaffoldScenariosPrompt({
+            count: opts.count,
+            contextMd,
+            examples,
+            axesAvailable,
+        }));
+    }
+    catch (err) {
+        if (err instanceof AgentTimeoutError)
+            log.warn(`agent stopped: ${err.message}`);
+        else
+            throw err;
+    }
+    finally {
+        monitor.stop();
+        if (convo)
+            await convo.end().catch(() => undefined);
+    }
+    const after = listScenarios(repoRoot);
+    const created = after.filter((id) => !before.has(id));
+    const allowed = new Set(axesAvailable.axisNames);
+    const validated = [];
+    const invalid = [];
+    for (const id of created) {
+        const v = validateScaffoldedScenario(repoRoot, id, allowed);
+        if (v.ok) {
+            validated.push(id);
+            log.ok(`  ${id} — validated`);
+        }
+        else {
+            invalid.push({ id, reason: v.reason });
+            log.error(`  ${id} — INVALID: ${v.reason}`);
+        }
+    }
+    log.info("");
+    if (created.length === 0) {
+        log.warn("agent produced 0 new scenarios. Re-run with --verbose to see what it did.");
+    }
+    else {
+        log.ok(`created ${created.length} scenario(s); ${validated.length} valid, ${invalid.length} invalid`);
+        if (invalid.length) {
+            log.warn("invalid scenarios were NOT deleted; inspect them under .agentrig/eval/scenarios/<id>/");
+            log.warn("then either fix them by hand or remove the directory and re-run --scaffold.");
+        }
+        log.info(color.dim("  Run a new scenario with: agentrig eval --dynamic --scenario <id>"));
+    }
+    return { created, validated, invalid };
+}
+//# sourceMappingURL=eval-scaffold.js.map

package/dist/commands/eval-scaffold.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-scaffold.js","sourceRoot":"","sources":["../../src/commands/eval-scaffold.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EACL,aAAa,EACb,cAAc,EACd,YAAY,EACZ,UAAU,GACX,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,4BAA4B,EAAE,cAAc,EAAwB,MAAM,qBAAqB,CAAC;AACzG,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAczC;kDACkD;AAClD,SAAS,eAAe,CAAC,QAAgB;IACvC,MAAM,GAAG,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,6BAA6B,CAAC,CAAC;IACzF,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,GAAG,CAAC,IAAI,CAAC;YACP,EAAE;YACF,WAAW,EAAE,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;YACzF,QAAQ,EAAE,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;YAChF,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;SACpF,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;2EAC2E;AAC3E,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,0BAA0B,CAAC,CAAC;IAC5D,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC/D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAA2D,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC;QAC1G,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;YACpD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;AACrD,CAAC;AAED;iFACiF;AACjF,SAAS,0BAA0B,CACjC,QAAgB,EAChB,EAAU,EACV,WAAwB;IAExB,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC3C,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,gDAAgD,EAAE,CAAC;IAC3F,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC;IAC9F,IAAI,EAAmC,CAAC;IACxC,IAAI,CAAC;QACH,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,6BAA8B,CAAW,CAAC,OAAO,EAAE,EAAE,CAAC;IACpF,CAAC;IACD,IAAI,MAAqC,CAAC;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,2BAA4B,CAAW,CAAC,OAAO,EAAE,EAAE,CAAC;IAClF,CAAC;IACD,iGAAiG;IACjG,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS;QACjC,GAAG,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC;QACzB,GAAG,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC;QACxB,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;KACpC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,IAAI,OAAO,CAAC,MAAM;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;IACxF,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;IACnF,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB,EAAE,IAAqB;IAC7E,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC;IACvC,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IACjE,GAAG,CAAC,EAAE,CAAC,gBAAgB,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;IAEtC,IAAI,IAAI,CAAC,aAAa,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;QACjD,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3D,IAAI,CAAC,CAAC,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,UAAU,IAAI,CAAC,aAAa,oBAAoB,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC,CAAC;IAC/F,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAC3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,oFAAoF;YACpF,+DAA+D,CAChE,CAAC;IACJ,CAAC;IACD,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IACrD,IAAI,aAAa,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,EAAE,sBAAsB,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACvF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,GAAG,CAAC,IAAI,CAAC,6EAA6E,CAAC,CAAC;QACxF,GAAG,CAAC,IAAI,CAAC,iFAAiF,CAAC,CAAC;IAC9F,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,KAAK,6BAA6B,CAAC,CAAC;IACjE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,uCAAuC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7H,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,qBAAqB,aAAa,CAAC,SAAS,CAAC,MAAM,MAAM,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAEhI,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC,KAAK,EAAE,CAAC;IAC9C,IAAI,KAAK,GAAkE,IAAI,CAAC;IAChF,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,QAAQ,CAAC,iBAAiB,CAAC;YACvC,GAAG,EAAE,QAAQ;YACb,aAAa,EAAE,cAAc;YAC7B,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5D,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,cAAc,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC3E,CAAC,CAAC;QACH,MAAM,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC;YAC5C,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS;YACT,QAAQ;YACR,aAAa;SACd,CAAC,CAAC,CAAC;IACN,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,iBAAiB;YAAE,GAAG,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;;YAC3E,MAAM,GAAG,CAAC;IACjB,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,IAAI,EAAE,CAAC;QACf,IAAI,KAAK;YAAE,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;IACjD,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,OAAO,GAAqC,EAAE,CAAC;IACrD,KAAK,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,0BAA0B,CAAC,QAAQ,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;QAC5D,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;YACT,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnB,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YACvC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACb,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,GAAG,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;IACxF,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,EAAE,CAAC,WAAW,OAAO,CAAC,MAAM,iBAAiB,SAAS,CAAC,MAAM,WAAW,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QACtG,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,uFAAuF,CAAC,CAAC;YAClG,GAAG,CAAC,IAAI,CAAC,6EAA6E,CAAC,CAAC;QAC1F,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,oEAAoE,CAAC,CAAC,CAAC;IAC5F,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;AACzC,CAAC"}