@turri-dev/cli 0.1.3-next.79e2ad0 → 0.1.3-next.b675516

package/dist/regenerate/conformance-agents.js

@@ -7,6 +7,7 @@ import { confinedTools } from "./confine.js";
 import { ensureInModulesScratch, modulesScratchDir } from "./paths.js";
 import { workspaceBlock } from "./workspace.js";
 import { reportActivity } from "./activity.js";
+import { runTiming, withTestingScripts } from "./test-scripts.js";
 import { createWebTools, WEB_TOOLS } from "./web.js";
 import { diffFiles, digestFiles } from "./codebase.js";
 import { ERROR_CAUSE_RULES, fixMethod, INTEGRITY_RULES, specPrimer } from "./prompt.js";
@@ -377,9 +378,17 @@ export function explorePrompt(buildDir, spec, scope, testingScripts, question, p
         ...(testingScripts !== undefined && testingScripts.length > 0
             ? [
                 "",
-                "This is the project's conformance testing script, which the harness runs to decide whether the",
-                "tests pass - read it for what it builds or installs before the tests run, and so what the tests can depend on:",
-                ...testingScripts.flatMap((script) => ["", `<script path="${script.path}" run-as="${script.runs}">`, script.content.trimEnd(), "</script>"]),
+                ...(testingScripts.some((script) => script.prepare === true)
+                    ? [
+                        "These are the project's prepare script, which the harness runs on the build first, and its conformance testing",
+                        "script, which runs only once the prepare script passes and decides whether the tests pass - read both for what",
+                        "they build or install before the tests run, and so what the tests can depend on:",
+                    ]
+                    : [
+                        "This is the project's conformance testing script, which the harness runs to decide whether the",
+                        "tests pass - read it for what it builds or installs before the tests run, and so what the tests can depend on:",
+                    ]),
+                ...testingScripts.flatMap((script) => ["", `<script${script.prepare === true ? ' kind="prepare environment"' : ""} path="${script.path}" run-as="${script.runs}">`, script.content.trimEnd(), "</script>"]),
             ]
             : []),
         ...(previous !== undefined && question === undefined
@@ -659,15 +668,28 @@ function words(text) {
  * in the specification - or when it hides why the software failed. A test asserting what the
  * specification leaves open is the test's mistake, however plausible the assertion.
  */
-export function unfoundedCodeVerdict(verdict, specText) {
+export function unfoundedCodeVerdict(verdict, specText, scriptText) {
     if (verdict.causeHidden === true)
         return undefined;
+    // What the conformance testing script needs of the build - a file, an entry point, a module - binds the code as the
+    // specification does: neither agent can change the script. Its own words, as they are in it.
+    const needs = verdict.scriptNeeds?.trim() ?? "";
+    if (needs !== "" && scriptText !== undefined) {
+        const flat = (text) => text.replace(/\s+/g, " ");
+        if (flat(needs).length >= 4 && flat(scriptText).includes(flat(needs)))
+            return undefined;
+        if (verdict.specSays === undefined) {
+            return (`Your verdict blames the code for what the conformance testing script needs, but "${needs}" is not in the script. ` +
+                "Call submit_verdict again: copy into script_needs the script's own words for what it needs of the build, exactly as they are in it.");
+        }
+    }
     const quote = verdict.specSays?.trim() ?? "";
     if (quote === "") {
         return ("Your verdict blames the code, but quotes no words of the specification it does not meet. Call submit_verdict " +
             "again: for code, put the specification's own words the code does not meet in spec_says, copied exactly from " +
-            "<specification>; or set cause_hidden when the output does not show why the software failed. If the specification " +
-            "does not say what the failing test asserts, the test is what is wrong: culprit tests.");
+            "<specification>; or, when the run fails on something the conformance testing script needs the build to have, " +
+            "its own words in script_needs; or set cause_hidden when the output does not show why the software failed. If " +
+            "the specification does not say what the failing test asserts, the test is what is wrong: culprit tests.");
     }
     const quoted = words(quote);
     if (quoted.trim().split(" ").length < 4 || !words(specText).includes(quoted)) {
@@ -677,6 +699,31 @@ export function unfoundedCodeVerdict(verdict, specText) {
     }
     return undefined;
 }
+/**
+ * Why a verdict's evidence cannot be taken as it is, or nothing when it can. Asked to weigh an escalation, the judge has
+ * to rest its verdict on what it found itself: each piece of evidence from a file it opened, the testing script or the
+ * failing run - and at least one not the run alone, which every judge before it read too. What the explorer's account
+ * says is not evidence: it was written before the run, and may be wrong.
+ */
+export function unfoundedEvidence(evidence, opened) {
+    const given = (evidence ?? []).filter((item) => item.source.trim() !== "" && item.shows.trim() !== "");
+    if (given.length === 0) {
+        return ("Your verdict gives no evidence. Call submit_verdict again with `evidence`: what you found yourself - in a file you " +
+            "opened, the testing script, or the failing run - each with where. What <how to invoke it> says is not evidence: it " +
+            "may be wrong, so check it against the testing script and the files.");
+    }
+    const inPrompt = (source) => /^(the )?(conformance )?testing script$|^(the )?prepare script$|^(the )?failing run$/i.test(source.trim());
+    const unread = given.filter((item) => !inPrompt(item.source) && !opened(item.source.trim()));
+    if (unread.length > 0) {
+        return (`Your evidence cites ${unread.map((item) => item.source.trim()).join(", ")}, which you did not open. Open each file you cite ` +
+            "and call submit_verdict again - evidence is what you found yourself, never what <how to invoke it> says.");
+    }
+    if (given.every((item) => /failing run$/i.test(item.source.trim()))) {
+        return ("Your evidence is only the failing run, which the judges before you read too. Check what it points at - the testing " +
+            "script, the code, the tests - and call submit_verdict again with what you found there.");
+    }
+    return undefined;
+}
 export async function judgeFailure(options) {
     let verdict;
     const submit = defineTool({
@@ -692,6 +739,13 @@ export async function judgeFailure(options) {
                 description: "For code: the specification's own words the code does not meet, copied exactly from <specification> - the sentence, or the part of it that says what the test checks",
             })),
             cause_hidden: Type.Optional(Type.Boolean({ description: "For code: true when it is wrong because the output does not show why the software failed - it hides the cause" })),
+            script_needs: Type.Optional(Type.String({
+                description: "For code: the conformance testing script's own words for what it needs the build to have - the file, entry point or module the run failed on - copied exactly from the script",
+            })),
+            evidence: Type.Optional(Type.Array(Type.Object({
+                source: Type.String({ description: "Where: the path of a file you opened, `testing script`, `prepare script` where there is one, or `failing run`" }),
+                shows: Type.String({ description: "What it shows, e.g. `the script runs node dist/cli.js` or `tsconfig.json emits src/cli.ts as dist/src/cli.js`" }),
+            }), { description: "What you found yourself, each with where - never what <how to invoke it> claims unchecked" })),
             functionalities: Type.Optional(Type.Array(Type.Object({
                 module: Type.String({ description: "The module the functionality is in" }),
                 id: Type.String({ description: "Its id in that module" }),
@@ -704,6 +758,8 @@ export async function judgeFailure(options) {
                 reason: given.reason,
                 ...(given.spec_says?.trim() ? { specSays: given.spec_says.trim() } : {}),
                 ...(given.cause_hidden === true ? { causeHidden: true } : {}),
+                ...(given.script_needs?.trim() ? { scriptNeeds: given.script_needs.trim() } : {}),
+                ...(given.evidence !== undefined && given.evidence.length > 0 ? { evidence: given.evidence } : {}),
                 ...(given.functionalities ? { functionalities: conflictRefs(given.functionalities) } : {}),
             };
             return { content: [{ type: "text", text: "Recorded." }], details: {}, terminate: true };
@@ -712,10 +768,27 @@ export async function judgeFailure(options) {
     // It reads both, and changes neither: the file tools and the web - to look up what a service it calls answers -
     // and no shell. It judges from the code, the tests, the spec and the failing run's output, which it is given.
     const fence = confinedTools({ workdir: options.conformanceDir, read: [options.buildDir, options.conformanceDir], write: [], noBash: true });
+    // What it opens - read, listed, searched - is what its evidence can cite.
+    const openedPaths = new Set();
+    const tools = fence.tools.map((tool) => ({
+        ...tool,
+        execute: (async (...args) => {
+            const path = args[1]?.path;
+            if (typeof path === "string" && path !== "")
+                openedPaths.add(resolve(options.conformanceDir, path));
+            return tool.execute(...args);
+        }),
+    }));
+    const opened = (cited) => {
+        // A file it opened, or one in a folder it listed or searched; cited as it was opened, or relative to the code or the tests.
+        const candidates = [resolve(options.conformanceDir, cited), resolve(options.buildDir, cited)];
+        return candidates.some((path) => [...openedPaths].some((seen) => path === seen || path.startsWith(`${seen}/`)));
+    };
+    const scriptText = options.testingScripts?.map((script) => script.content).join("\n");
     const before = { code: digestFiles(options.buildDir), tests: digestFiles(options.conformanceDir) };
     const session = await createExecutor({
         cwd: options.conformanceDir,
-        customTools: [submit, ...createWebTools(), ...fence.tools],
+        customTools: [submit, ...createWebTools(), ...tools],
         tools: [...fence.names, ...WEB_TOOLS, "submit_verdict"],
         model: options.model,
         label: `${options.label} judge`, role: "judge",
@@ -730,8 +803,15 @@ export async function judgeFailure(options) {
                 await ask(session, "You have not called submit_verdict. Call it now: code, tests, spec or environment, and why.", options.onDelta);
                 continue;
             }
+            // Asked to weigh an escalation, a verdict of code or tests rests on what it found itself.
+            const unproven = options.escalation !== undefined && (verdict.culprit === "code" || verdict.culprit === "tests") ? unfoundedEvidence(verdict.evidence, opened) : undefined;
+            if (unproven !== undefined) {
+                verdict = undefined;
+                await ask(session, unproven, options.onDelta);
+                continue;
+            }
             if (verdict.culprit === "code") {
-                const unfounded = unfoundedCodeVerdict(verdict, options.spec.text);
+                const unfounded = unfoundedCodeVerdict(verdict, options.spec.text, scriptText);
                 if (unfounded === undefined)
                     break;
                 verdict = undefined;
@@ -759,18 +839,26 @@ export async function judgeFailure(options) {
     }
     if (verdict?.culprit === "spec" && specConflictProblem(verdict.functionalities, options.scope) !== undefined)
         verdict = undefined;
-    if (verdict?.culprit === "code" && unfoundedCodeVerdict(verdict, options.spec.text) !== undefined) {
+    // An escalation weighed without evidence escalates nothing: the tests stay the conformance agent's, as before it.
+    if (options.escalation !== undefined && verdict?.culprit === "code" && unfoundedEvidence(verdict.evidence, opened) !== undefined) {
+        return { culprit: "tests", reason: `${verdict.reason}\n\nBut the judge gave no evidence it found itself that the code is what is wrong, so the tests are looked at again.` };
+    }
+    if (verdict?.culprit === "code" && unfoundedCodeVerdict(verdict, options.spec.text, scriptText) !== undefined) {
         // Blaming the code takes the specification's own words; without them, the test asserts what the spec does not say.
         return { culprit: "tests", reason: `${verdict.reason}\n\nBut no words of the specification say so: a test that asserts what the specification does not say is what is wrong. Check each failing assertion against the specification, and assert only what it says.` };
     }
     if (verdict?.culprit === "code" && verdict.specSays !== undefined) {
         verdict = { ...verdict, reason: `${verdict.reason}\n\nThe specification says: "${verdict.specSays}"` };
     }
+    if (verdict?.culprit === "code" && verdict.scriptNeeds !== undefined) {
+        verdict = { ...verdict, reason: `${verdict.reason}\n\nThe conformance testing script needs: "${verdict.scriptNeeds}"` };
+    }
     // No verdict is no evidence the code is wrong: the tests are the newer work, and looked at first.
     return verdict ?? { culprit: "tests", reason: "The judge gave no verdict that could be used; check the failing tests against the specification first." };
 }
 export function judgePrompt(options) {
     const tested = options.scope.tested;
+    const scripts = options.testingScripts ?? [];
     return [
         specPrimer({ testReqs: true }),
         `The conformance tests of the \`${options.spec.moduleName}\` specification failed. Decide what is wrong: the code, the tests, the specification itself, or the run's environment.`,
@@ -780,7 +868,9 @@ export function judgePrompt(options) {
         "The specification is the judge, never either of them:",
         "- tests: a test asserts something the specification does not say, misreads it, or reaches the software the",
         "  wrong way - a wrong command, route or setup (see <how to invoke it>).",
-        "- code: the code does not do what the specification says, and the test checks exactly that.",
+        "- code: the code does not do what the specification says, and the test checks exactly that. Or the run fails",
+        "  on something the conformance testing script needs the code to provide - a build output, an entry point, a",
+        "  module - and the code does not provide it: quote the script's own words for it in script_needs.",
         "  Only what the specification says counts: anything it leaves open - a name, coordinates, a version, a path,",
         "  a package, a port, a file layout - is what the code already has. The tests follow it, and it is never",
         "  code - changing the code to fit a test's guess moves what every other test points at.",
@@ -801,6 +891,8 @@ export function judgePrompt(options) {
         "  the tests set up themselves - a credential, a variable or a fixture a test sets, overrides or names wrong: that is",
         "  tests; nor for a toolchain or package missing where the tests run: the agents install those. The run stops on it",
         `  once it has been judged so ${ENVIRONMENT_RAISES} times in a row; until then the tests are checked for whether they cause it.`,
+        "  A run the time limit stopped is environment only when nothing hangs and it needs more time than a test run may",
+        "  take - and then it stops at once.",
         "",
         "When the output does not show why the software failed - it answered with an error that names no cause, and",
         "logged nothing about it - the code is what is wrong first: it hides the cause, and nobody can fix what they",
@@ -810,6 +902,25 @@ export function judgePrompt(options) {
         "verdict of code without them is not taken. What the specification leaves open is never the code's fault: a test",
         "that asserts it - a status code, a message, a format the specification does not name - is what is wrong.",
         "",
+        ...(scripts.length > 0
+            ? [
+                "The conformance testing script below is how every run is made, and neither agent can change it: it is never",
+                "what is wrong. What it does with the tests, the tests have to fit; what it needs of the code - a file it runs or",
+                "checks for, a command it builds with - the code has to provide. A verdict that the script is wrong is neither",
+                "code nor tests: say which of the two has to change to get past it.",
+                ...(scripts.some((script) => script.prepare === true)
+                    ? [
+                        "The prepare script below it runs on the build before every run, and the conformance testing script only once it",
+                        "passes: it binds the same way - neither agent can change it, and what it needs of the code the code provides.",
+                    ]
+                    : []),
+                "",
+            ]
+            : []),
+        "<how to invoke it> is the explorer's account, written before the run, and it can be wrong - a folder, a path, a",
+        "command. Never rest a verdict on it: check each claim you use against the testing script and the files, and give",
+        "in `evidence` what you found yourself, each with where.",
+        "",
         `You have ${JUDGE_CALLS} tool calls to decide in: read what the failure points at, not the whole codebase.`,
         "",
         "When the code and the tests could both be what is wrong, choose tests. When both are wrong, say which to fix",
@@ -832,11 +943,63 @@ export function judgePrompt(options) {
         "",
         `<how to invoke it>\n${options.invocation.trim()}\n</how to invoke it>`,
         "",
+        ...scripts.flatMap((script) => {
+            const tag = script.prepare === true ? "prepare script" : "conformance testing script";
+            return [`<${tag} path="${script.path}" run-as="${script.runs}">`, script.content.trimEnd(), `</${tag}>`, ""];
+        }),
+        ...(options.run.timedOut === true ? [...timedOutBlock(options.run), ""] : []),
         `<failing run exit="${options.run.exitCode}" command="${options.run.command}">`,
+        runTiming(options.run),
+        "",
         options.run.output.trimEnd(),
         "</failing run>",
+        ...(options.escalation !== undefined ? ["", ...escalationBlock(options.escalation)] : []),
     ].join("\n");
 }
+/**
+ * For the judge, of a run the time limit stopped: what kept it from finishing - something that hangs, the code's or a
+ * test's, or a suite that is only slow, which no agent can give more time.
+ */
+export function timedOutBlock(run) {
+    const limit = Math.round(run.timeoutMs / 1000);
+    return [
+        "<timed out>",
+        `This run did not end on its own: the time limit stopped it at ${limit}s, and it counts as failed. Its output ends where it`,
+        "was stopped - the step running then, or the one after the last it printed, is what did not finish. Decide what kept it:",
+        "- code: the software hangs - it waits on input, keeps a server in the foreground, never exits, loops, or blocks on a",
+        "  lock or a call that never answers - when the tests reach it as the specification says.",
+        "- tests: a test waits for something that never comes - a process it started and never stops, output it waits for that",
+        "  the software never prints, a port or a file nobody makes, a sleep or a retry with no end.",
+        "- environment: nothing hangs - the output shows the run making steady progress right up to the limit (installing,",
+        "  building, test after test passing) and it needs more time than a test run may take. No agent can give it that: the",
+        "  run stops at once, and the person is told to raise the limit. Say from the output how far it got, and what took",
+        "  the time.",
+        "Judge from the output and the files, never from the time alone.",
+        "</timed out>",
+    ];
+}
+/** For the judge: the tests were fixed for this failure, and it did not go away - is it the code's? */
+function escalationBlock(escalation) {
+    const changed = escalation.changed.length === 0 ? "changed no file" : `changed ${escalation.changed.slice(0, 8).join(", ")}${escalation.changed.length > 8 ? ` and ${escalation.changed.length - 8} more` : ""}`;
+    return [
+        "<escalation>",
+        "This failure was judged the tests' before, and the conformance agent - who can change only the tests - was sent to",
+        `fix it. It ${changed}, and the run fails the same way. What it was told:`,
+        "",
+        escalation.reason.trim(),
+        "",
+        "Decide again, now whether the failure belongs to the code rather than the tests. It is the code's when the run",
+        "fails on something the code is meant to produce - a build output, an entry point, a module the testing script needs",
+        "- or when the tests ran and what the software does is not what the specification says. It stays the tests' when a",
+        "test is still wrong and the code is right: say exactly what in the tests, so the next fix is not the last one again.",
+        "",
+        "Gather the evidence yourself: open the files the failure points at - the code's build configuration and what it",
+        "emits, the tests - and read the testing script. Give it in `evidence`: a verdict of code or tests without evidence",
+        "you found is not taken. The coding agent is handed your reason and evidence: say what the code has to provide, and",
+        "where it now falls short (e.g. `the script runs dist/cli.js; the build emits dist/src/cli.js`).",
+        "</escalation>",
+    ];
+}
 /**
  * The coding agent a conformance run sends the code to, when the judge says the code is wrong: it
  * works in the code folder, may read the tests but never changes them, and keeps the unit
@@ -858,7 +1021,10 @@ export async function createCodeFixer(options) {
         model: options.model,
         label: `${options.label} coding agent`, role: "coding",
         system: `${fence.note}\n\n${workspaceBlock({ dir: options.buildDir, hasMyDiff: options.toolNames.includes("my_diff") })}`,
-        task: codeFixerTask(options),
+        // What the conformance testing script needs of the build, the code provides: the fixer reads it, not only a judge's quote of it.
+        task: options.testingScripts !== undefined
+            ? withTestingScripts(codeFixerTask(options), options.testingScripts.scripts, ["conformance", "unit"], options.testingScripts.root)
+            : codeFixerTask(options),
         compaction: { instructions: CODER_COMPACTION },
         ...(options.checkEnvironment !== undefined ? { checkEnvironment: options.checkEnvironment } : {}),
     });
@@ -911,12 +1077,53 @@ export function codeFixPrompt(options) {
         `<why the code>\n${options.reason.trim()}\n</why the code>`,
         "",
         `<failing run command="${options.run.command}">`,
+        runTiming(options.run),
+        "",
         options.run.output.trimEnd(),
         "</failing run>",
         "",
         "Fix the code, run the conformance tests and the unit tests, and finish.",
     ].join("\n");
 }
+/** How much of the conformance tests an escalation hands the coding agent, whole: the rest it reads itself. */
+const ESCALATED_TESTS_CHARS = 24_000;
+/**
+ * An escalated failure for the code fixer: the tests were fixed for it and it did not go away, and the judge found it
+ * the code's. It is handed the conformance testing script's error, the tests as they stand and the judge's evidence.
+ */
+export function escalatedFixPrompt(options) {
+    let room = ESCALATED_TESTS_CHARS;
+    const shown = options.tests.flatMap((test) => {
+        if (room <= 0)
+            return [`<conformance test path="${test.path}" shown="no - read it yourself" />`];
+        const content = test.content.length > room ? `${test.content.slice(0, room)}\n... (cut short - read the rest yourself)` : test.content;
+        room -= content.length;
+        return [`<conformance test path="${test.path}">`, content.trimEnd(), "</conformance test>"];
+    });
+    return [
+        `The conformance tests failed (exit ${options.run.exitCode}). The conformance agent was sent to fix the tests for it, and`,
+        "the run failed the same way. A judge then looked again, gathered the evidence below itself, and found that the",
+        "failure belongs to the code:",
+        "",
+        `<why the code>\n${options.reason.trim()}\n</why the code>`,
+        "",
+        "<evidence>",
+        ...options.evidence.map((item) => `- ${item.source.trim()}: ${item.shows.trim()}`),
+        "</evidence>",
+        "",
+        `<failing run command="${options.run.command}">`,
+        runTiming(options.run),
+        "",
+        options.run.output.trimEnd(),
+        "</failing run>",
+        "",
+        "The conformance tests as they stand - not yours to change:",
+        ...shown,
+        "",
+        "Fix the code, run the conformance tests and the unit tests, and finish. This is the last attempt at this failure:",
+        "if the run still fails the same way after your fix, the work stops.",
+    ].join("\n");
+}
 /** The run's environment judged what is wrong: the conformance agent checks the tests do not cause it, and changes nothing when they do not. */
 export function environmentCheckPrompt(options) {
     return [
@@ -944,6 +1151,8 @@ export function testsFixPrompt(options) {
         `<why the tests>\n${options.reason.trim()}\n</why the tests>`,
         "",
         "<failing run>",
+        runTiming(options.run),
+        "",
         options.run.output.trimEnd(),
         "</failing run>",
         "",

package/dist/regenerate/conformance.js

@@ -9,14 +9,14 @@ import { abandonWork, diffSinceSnapshot, moduleDirOf, recordHistory, startWork,
 import { createWorkLedger, describeAccounts, SUBMIT_WORK_NOTE, unanswered, unansweredPrompt } from "./submit-work.js";
 import { CONFORMANCE_COMPACTION } from "./compaction.js";
 import { NOTES_TURN, notesForSession, passedOutcome, readNotes, reflectOnReview, withNotes } from "./notes.js";
-import { labelsPrompt, labelsPromptFor, misplacedConformanceTests, misplacedTestsPrompt, missingTestLabels } from "./test-kinds.js";
+import { acceptanceLabel, acceptanceText, labelsPrompt, labelsPromptFor, misplacedConformanceTests, misplacedTestsPrompt, missingTestLabels } from "./test-kinds.js";
 import { createScriptTool, findTestScripts, projectRootFor, requireTestScripts, runConformanceScript, runUnitScript, failureSignature, STUCK_ROUNDS, scriptFailure, staticChecksOnly, withTestingScripts, } from "./test-scripts.js";
 import { createWebTools, WEB_TOOLS } from "./web.js";
 import { createMyDiffTool } from "./self-check.js";
 import { createServiceTools } from "./services.js";
-import { missingToolchainError, toolchainError } from "./environment-error.js";
+import { missingToolchainError, timeLimitError, toolchainError } from "./environment-error.js";
 import { workspaceBlock } from "./workspace.js";
-import { codeFixPrompt, conflictKey, createCodeFixer, answerAsExplorer, createAskExplorerTool, exploreInvocation, judgeFailure, SPEC_CONFLICT_RAISES, ENVIRONMENT_RAISES, environmentCheckPrompt, createTestReviewer, testReviewRequest, reviewFeedbackPrompt, testsFixPrompt, sortedRefs, } from "./conformance-agents.js";
+import { codeFixPrompt, conflictKey, escalatedFixPrompt, createCodeFixer, answerAsExplorer, createAskExplorerTool, exploreInvocation, judgeFailure, SPEC_CONFLICT_RAISES, ENVIRONMENT_RAISES, environmentCheckPrompt, createTestReviewer, testReviewRequest, reviewFeedbackPrompt, testsFixPrompt, sortedRefs, } from "./conformance-agents.js";
 import { resolveRequiresChain } from "./requires.js";
 import { notesToRead, recordNote } from "./notes.js";
 import { reportSpecConflict } from "./activity.js";
@@ -157,9 +157,11 @@ export async function renderConformanceTests(options) {
     // may read the code but writes only tests. An explorer reads the code first and writes down how the
     // software is invoked, which is what the tests are written against.
     const label = `${moduleName} ${options.diff !== undefined ? "spec change" : frid}`;
-    const specContext = { moduleName, text: specText(specifications, moduleName, frids, required, options.change ?? options.diff, sectionModules(tree, moduleName)) };
     // For a spec change: the functionalities it touches, whose acceptance tests are labelled, checked and reviewed.
     const touched = forChange ? touchedFunctionalities(tree, frids, options.diff) : [];
+    // The acceptance tests the run's tests carry out: every functionality the change touches, or the one under test.
+    const carriedOut = forChange ? touched : [{ frid: frid, acceptanceTests: specifications[ACCEPTANCE_TESTS] ?? [] }];
+    const specContext = { moduleName, text: specText(specifications, moduleName, frids, required, carriedOut, options.change ?? options.diff, sectionModules(tree, moduleName)) };
     // Any functionality of the codebase - this module's, and those of the modules it requires. A
     // functionality's run may raise a spec conflict only with that functionality; a spec diff's, between any two.
     const own = (forChange ? frids : frids.slice(0, frids.indexOf(frid) + 1)).map((id) => ({ module: moduleName, frid: id }));
@@ -194,6 +196,11 @@ export async function renderConformanceTests(options) {
     let environmentRaises = 0;
     /** How many fixes in a row - the code's or the tests' - changed no file: an agent that will not, or cannot, act on the verdict. */
     let idleFixes = 0;
+    /** The last fix was the tests', for a failure judged theirs: should the run fail the same way again, the judge weighs escalating it. */
+    let testsFix;
+    /** A failure escalated to the code: should the run still fail the same way after the code's fix, the run stops. */
+    let escalated;
+    let escalation;
     /**
      * One attempt at the failing run, written into the notes' pitfalls - what the next judge reads
      * first - as cause and effect: what failed, how it was judged, and what was changed; the next
@@ -218,11 +225,15 @@ export async function renderConformanceTests(options) {
     };
     stage(`exploring the code for how it is invoked${options.model ? ` (${options.model})` : ""}`);
     // The explorer reads the conformance testing script too: it is how the tests are built and run, and so how they reach the software.
+    // The prepare script, where there is one, runs on the build before it - and only once it passes does the suite run.
     const testingScripts = [
+        ...(scripts.prepare !== undefined
+            ? [{ path: relative(root, scripts.prepare), runs: "<script> <copy of the build>, before every run of the conformance testing script", content: readFileSync(scripts.prepare, "utf8"), prepare: true }]
+            : []),
         { path: relative(root, scripts.conformance), runs: "<script> <copy of the build> <copy of the whole conformance suite>", content: readFileSync(scripts.conformance, "utf8") },
     ];
     // The acceptance tests the explorer's plan has to carry out, step by step.
-    const toCarryOut = (forChange ? touched : [{ frid: frid, acceptanceTests: specifications[ACCEPTANCE_TESTS] ?? [] }])
+    const toCarryOut = carriedOut
         .map((item) => ({ module: moduleName, frid: item.frid, count: item.acceptanceTests.length }))
         .filter((item) => item.count > 0);
     let invocation = await exploreInvocation({ buildDir, spec: specContext, scope, testingScripts, acceptance: toCarryOut, label, model: options.model, onDelta: options.onDelta });
@@ -314,6 +325,7 @@ export async function renderConformanceTests(options) {
             ],
             toolNames: ["run_unit_tests", "run_conformance_tests", ...fixerServices.names, "update_notes"],
             checkEnvironment: { run: () => runUnitScript(buildDir, scripts.unit), script: scripts.unit },
+            testingScripts: { scripts, root },
         });
         fixerConformanceRuns.newTurn();
         return ask(fixer, withNotes(text, buildDir), options.onDelta);
@@ -414,6 +426,15 @@ export async function renderConformanceTests(options) {
                     abandonWork(start, moduleName, "the conformance testing script could not run");
                 throw lacking;
             }
+            // A failure escalated to the code, still the same after the code's fix: neither agent could fix it, and the run stops
+            // with why the judge found it the code's. Anything else it fails on is a new failure, judged as any other.
+            if (escalated !== undefined && recheck.exitCode !== 0 && failureSignature(recheck) === escalated.signature) {
+                escalation = { reason: escalated.reason, evidence: escalated.evidence, error: failureLine(recheck), failure: truncateOutput(recheck.output, undefined).slice(-4000) };
+                stage("the failure escalated to the code is still there after the coding agent's fix - stopping");
+                recordNote(conformanceDir, "pitfalls", `stopped after ${rounds} attempts: failed with ${failureLine(recheck)}, escalated to the code - ${escalated.reason.replace(/\s+/g, " ").slice(0, 240)} - and the code's fix did not change it`, `${label} conformance`);
+                break;
+            }
+            escalated = undefined;
             if (recheck.exitCode !== 0) {
                 // Nothing changing: the same failure, taken the same way, round after round. Once, the explorer
                 // is asked what the tests lack - most often how they reach the software; after that, it stops.
@@ -480,19 +501,6 @@ export async function renderConformanceTests(options) {
                         idleFixes = 0;
                     continue;
                 }
-                // A run out of time is not a question of test or code: the tests' environment is the conformance agent's.
-                if (recheck.timedOut === true) {
-                    const moved = await unmoved();
-                    if (moved === "stuck")
-                        break;
-                    if (moved === "asked")
-                        continue;
-                    stage("the conformance run did not get to the tests - back to the conformance agent");
-                    const fixed = await changing(() => askConformance(scriptFailure(recheck, "conformance testing"), options.onDelta));
-                    response = fixed.result;
-                    recordAttempt(rounds, recheck, `not a question of code or tests (${recheck.timedOut === true ? "the time limit" : `exit ${recheck.exitCode}`})`, { who: "conformance agent", files: fixed.files });
-                    continue;
-                }
                 stage(`conformance tests failing - working out whether the code, the tests or the spec are wrong${judgeModel ? ` (${judgeModel})` : ""}`);
                 // A fix that changed no file leaves the next run as this one: IDLE_FIXES of them in a row, and nothing will change.
                 const idle = (files) => {
@@ -504,6 +512,12 @@ export async function renderConformanceTests(options) {
                     recordNote(conformanceDir, "pitfalls", `stuck after ${rounds} attempts: failed with ${failureLine(recheck)}, and ${IDLE_FIXES} fixes in a row changed nothing`, `${label} conformance`);
                     return true;
                 };
+                // The tests were fixed for this very failure, and it is still here: the judge weighs whether it is the code's.
+                const signature = failureSignature(recheck);
+                const escalating = testsFix !== undefined && testsFix.signature === signature ? { reason: testsFix.reason, changed: testsFix.files } : undefined;
+                testsFix = undefined;
+                if (escalating !== undefined)
+                    stage("the tests' fix did not make the failure go away - the judge weighs whether it is the code's");
                 const verdict = await judgeFailure({
                     buildDir,
                     conformanceDir,
@@ -513,11 +527,22 @@ export async function renderConformanceTests(options) {
                     scope,
                     // The tests' notes - every attempt so far - and the code's: it weighs the one against the other.
                     notes: [notesToRead(conformanceDir), notesToRead(buildDir)].filter((part) => part !== "").join("\n\n"),
+                    // The script itself, so the judge checks the explorer's account against it rather than repeating it.
+                    testingScripts,
+                    ...(escalating !== undefined ? { escalation: escalating } : {}),
                     label,
                     model: judgeModel,
                     onDelta: options.onDelta,
                 });
                 const reason = verdict.reason.split("\n")[0].slice(0, 160);
+                // Stopped by the time limit while it was still getting on: no agent can give it more time. The pulse stops now -
+                // rolled back, no retry - and the person is told how to raise the limit.
+                if (verdict.culprit === "environment" && recheck.timedOut === true) {
+                    stage(`the conformance run needs more than the ${Math.round(recheck.timeoutMs / 1000)}s a test run may take - stopping`);
+                    if (start !== undefined)
+                        abandonWork(start, moduleName, "the conformance testing script needs more time than a test run may take");
+                    throw timeLimitError(recheck, { moduleName, script: scripts.conformance, kind: "conformance", reason: verdict.reason });
+                }
                 // The environment - what the run is given - is the person's to fix: once judged so twice in a row, the run stops.
                 if (verdict.culprit === "environment") {
                     environmentRaises += 1;
@@ -585,15 +610,34 @@ export async function renderConformanceTests(options) {
                     recordAttempt(rounds, recheck, `a spec conflict between functionalities ${key}: ${verdict.reason}`, { who: "conformance agent", files: fixed.files });
                 }
                 else if (verdict.culprit === "code") {
-                    reportActivity(`  the code: ${reason}`);
-                    stage("the code is wrong - the coding agent fixes it");
-                    // One fixer for the run, starting each fix from what the ones before it tried: its work is compacted as
-                    // it gets longer, so a session carried through the run does not re-read every earlier fix and suite output
-                    // whole.
-                    const fixed = await changing(() => askFixer(codeFixPrompt({ run: recheck, reason: verdict.reason })));
-                    recordAttempt(rounds, recheck, `the code wrong: ${verdict.reason}`, { who: "coding agent", files: fixed.files });
-                    if (idle(fixed.files))
-                        break;
+                    if (escalating !== undefined) {
+                        // Escalated: the coding agent is handed the script's error, the tests as they stand and the judge's evidence.
+                        // Its fix is the last attempt at this failure - should the run fail the same way after it, the run stops.
+                        const evidence = verdict.evidence ?? [];
+                        reportActivity(`  escalated to the code: ${reason}`);
+                        stage("the judge finds the failure the code's - escalated to the coding agent");
+                        const tests = listFiles(conformanceDir).map((path) => ({ path, content: readFileSync(join(conformanceDir, path), "utf8") }));
+                        const fixed = await changing(() => askFixer(escalatedFixPrompt({ run: recheck, reason: verdict.reason, evidence, tests })));
+                        recordAttempt(rounds, recheck, `escalated to the code: ${verdict.reason}`, { who: "coding agent", files: fixed.files });
+                        if (fixed.files.length === 0) {
+                            escalation = { reason: verdict.reason, evidence, error: failureLine(recheck), failure: truncateOutput(recheck.output, undefined).slice(-4000), changedNothing: true };
+                            stage("the coding agent changed nothing for the failure escalated to it - stopping");
+                            break;
+                        }
+                        idleFixes = 0;
+                        escalated = { signature, reason: verdict.reason, evidence };
+                    }
+                    else {
+                        reportActivity(`  the code: ${reason}`);
+                        stage("the code is wrong - the coding agent fixes it");
+                        // One fixer for the run, starting each fix from what the ones before it tried: its work is compacted as
+                        // it gets longer, so a session carried through the run does not re-read every earlier fix and suite output
+                        // whole.
+                        const fixed = await changing(() => askFixer(codeFixPrompt({ run: recheck, reason: verdict.reason })));
+                        recordAttempt(rounds, recheck, `the code wrong: ${verdict.reason}`, { who: "coding agent", files: fixed.files });
+                        if (idle(fixed.files))
+                            break;
+                    }
                     // The fix has to build and pass its own unit tests before the suite runs again: a build
                     // error is the coding agent's to fix, not a judge's to work out from the suite's output.
                     for (let again = 0; again < FIX_UNIT_CHECKS; again += 1) {
@@ -613,7 +657,11 @@ export async function renderConformanceTests(options) {
                     const fixed = await changing(() => askConformance(testsFixPrompt({ run: recheck, reason: verdict.reason }), options.onDelta));
                     response = fixed.result;
                     recordAttempt(rounds, recheck, `the tests wrong: ${verdict.reason}`, { who: "conformance agent", files: fixed.files });
-                    if (idle(fixed.files))
+                    testsFix = { signature, reason: verdict.reason, files: fixed.files };
+                    // A fix that changed nothing does not stop the run before the judge has weighed escalating the failure.
+                    if (escalating === undefined)
+                        idleFixes = fixed.files.length === 0 ? idleFixes + 1 : 0;
+                    else if (idle(fixed.files))
                         break;
                 }
                 continue;
@@ -646,7 +694,7 @@ export async function renderConformanceTests(options) {
                     buildDir,
                     conformanceDir,
                     spec: specContext,
-                    functionalities: forChange ? touched : [{ frid: frid, acceptanceTests: specifications[ACCEPTANCE_TESTS] ?? [] }],
+                    functionalities: carriedOut,
                     forChange,
                     invocation,
                     label,
@@ -772,6 +820,7 @@ export async function renderConformanceTests(options) {
             ...(specConflict !== undefined ? { specConflict } : {}),
             ...(stuck !== undefined ? { stuck } : {}),
             ...(environment !== undefined ? { environment } : {}),
+            ...(escalation !== undefined ? { escalation } : {}),
             ...(unlabelled !== undefined ? { unlabelled } : {}),
             ...(reviewWaived !== undefined ? { reviewWaived } : {}),
             ...(verified && green !== undefined ? { green } : {}),
@@ -905,9 +954,10 @@ export function failureLine(run) {
 }
 /**
  * The spec as the explorer, the judge and the coding agent are shown it - its functional specs
- * numbered with their ids, so a spec conflict can name them.
+ * numbered with their ids, so a spec conflict can name them, and the acceptance tests the run carries
+ * out labelled as their tests are, so a verdict can quote them.
  */
-function specText(specifications, moduleName, frids, required, diff, modules) {
+function specText(specifications, moduleName, frids, required, acceptance, diff, modules) {
     // Every functionality in the codebase, each with its module: the required modules' first, then this one's.
     const numbered = (key, items) => key === FUNCTIONAL_REQUIREMENTS
         ? [
@@ -915,10 +965,14 @@ function specText(specifications, moduleName, frids, required, diff, modules) {
         ]
         : items;
     // Each section in its tag, as every agent is shown it (see specPrimer).
-    const sections = [DEFINITIONS, NON_FUNCTIONAL_REQUIREMENTS, TEST_REQUIREMENTS, FUNCTIONAL_REQUIREMENTS, ACCEPTANCE_TESTS]
+    const sections = [DEFINITIONS, NON_FUNCTIONAL_REQUIREMENTS, TEST_REQUIREMENTS, FUNCTIONAL_REQUIREMENTS]
         .map((key) => [key, specifications[key] ?? []])
         .filter(([, items]) => items.length > 0)
-        .map(([key, items]) => (key === FUNCTIONAL_REQUIREMENTS || key === ACCEPTANCE_TESTS ? section(key, numbered(key, items)) : moduleSection(key, items, moduleName, modules)));
+        .map(([key, items]) => (key === FUNCTIONAL_REQUIREMENTS ? section(key, numbered(key, items)) : moduleSection(key, items, moduleName, modules)));
+    // Not only the last functionality's: a spec change's run carries out the acceptance tests of every one it touches.
+    const tests = acceptance.flatMap((item) => item.acceptanceTests.map((test, index) => `${acceptanceLabel(item.frid, index + 1)}: ${acceptanceText(test)}`));
+    if (tests.length > 0)
+        sections.push(section(ACCEPTANCE_TESTS, tests));
     if (diff !== undefined)
         sections.push(specChangeBlock(diff));
     return sections.join("\n\n");