@turri-dev/cli 0.1.3-next.a5d3797 → 0.1.3-next.b675516

package/dist/regenerate/conformance-agents.js

@@ -7,6 +7,7 @@ import { confinedTools } from "./confine.js";
 import { ensureInModulesScratch, modulesScratchDir } from "./paths.js";
 import { workspaceBlock } from "./workspace.js";
 import { reportActivity } from "./activity.js";
+import { runTiming, withTestingScripts } from "./test-scripts.js";
 import { createWebTools, WEB_TOOLS } from "./web.js";
 import { diffFiles, digestFiles } from "./codebase.js";
 import { ERROR_CAUSE_RULES, fixMethod, INTEGRITY_RULES, specPrimer } from "./prompt.js";
@@ -377,9 +378,17 @@ export function explorePrompt(buildDir, spec, scope, testingScripts, question, p
         ...(testingScripts !== undefined && testingScripts.length > 0
             ? [
                 "",
-                "This is the project's conformance testing script, which the harness runs to decide whether the",
-                "tests pass - read it for what it builds or installs before the tests run, and so what the tests can depend on:",
-                ...testingScripts.flatMap((script) => ["", `<script path="${script.path}" run-as="${script.runs}">`, script.content.trimEnd(), "</script>"]),
+                ...(testingScripts.some((script) => script.prepare === true)
+                    ? [
+                        "These are the project's prepare script, which the harness runs on the build first, and its conformance testing",
+                        "script, which runs only once the prepare script passes and decides whether the tests pass - read both for what",
+                        "they build or install before the tests run, and so what the tests can depend on:",
+                    ]
+                    : [
+                        "This is the project's conformance testing script, which the harness runs to decide whether the",
+                        "tests pass - read it for what it builds or installs before the tests run, and so what the tests can depend on:",
+                    ]),
+                ...testingScripts.flatMap((script) => ["", `<script${script.prepare === true ? ' kind="prepare environment"' : ""} path="${script.path}" run-as="${script.runs}">`, script.content.trimEnd(), "</script>"]),
             ]
             : []),
         ...(previous !== undefined && question === undefined
@@ -703,7 +712,7 @@ export function unfoundedEvidence(evidence, opened) {
             "opened, the testing script, or the failing run - each with where. What <how to invoke it> says is not evidence: it " +
             "may be wrong, so check it against the testing script and the files.");
     }
-    const inPrompt = (source) => /^(the )?(conformance )?testing script$|^(the )?failing run$/i.test(source.trim());
+    const inPrompt = (source) => /^(the )?(conformance )?testing script$|^(the )?prepare script$|^(the )?failing run$/i.test(source.trim());
     const unread = given.filter((item) => !inPrompt(item.source) && !opened(item.source.trim()));
     if (unread.length > 0) {
         return (`Your evidence cites ${unread.map((item) => item.source.trim()).join(", ")}, which you did not open. Open each file you cite ` +
@@ -734,7 +743,7 @@ export async function judgeFailure(options) {
                 description: "For code: the conformance testing script's own words for what it needs the build to have - the file, entry point or module the run failed on - copied exactly from the script",
             })),
             evidence: Type.Optional(Type.Array(Type.Object({
-                source: Type.String({ description: "Where: the path of a file you opened, `testing script`, or `failing run`" }),
+                source: Type.String({ description: "Where: the path of a file you opened, `testing script`, `prepare script` where there is one, or `failing run`" }),
                 shows: Type.String({ description: "What it shows, e.g. `the script runs node dist/cli.js` or `tsconfig.json emits src/cli.ts as dist/src/cli.js`" }),
             }), { description: "What you found yourself, each with where - never what <how to invoke it> claims unchecked" })),
             functionalities: Type.Optional(Type.Array(Type.Object({
@@ -882,6 +891,8 @@ export function judgePrompt(options) {
         "  the tests set up themselves - a credential, a variable or a fixture a test sets, overrides or names wrong: that is",
         "  tests; nor for a toolchain or package missing where the tests run: the agents install those. The run stops on it",
         `  once it has been judged so ${ENVIRONMENT_RAISES} times in a row; until then the tests are checked for whether they cause it.`,
+        "  A run the time limit stopped is environment only when nothing hangs and it needs more time than a test run may",
+        "  take - and then it stops at once.",
         "",
         "When the output does not show why the software failed - it answered with an error that names no cause, and",
         "logged nothing about it - the code is what is wrong first: it hides the cause, and nobody can fix what they",
@@ -897,6 +908,12 @@ export function judgePrompt(options) {
                 "what is wrong. What it does with the tests, the tests have to fit; what it needs of the code - a file it runs or",
                 "checks for, a command it builds with - the code has to provide. A verdict that the script is wrong is neither",
                 "code nor tests: say which of the two has to change to get past it.",
+                ...(scripts.some((script) => script.prepare === true)
+                    ? [
+                        "The prepare script below it runs on the build before every run, and the conformance testing script only once it",
+                        "passes: it binds the same way - neither agent can change it, and what it needs of the code the code provides.",
+                    ]
+                    : []),
                 "",
             ]
             : []),
@@ -926,13 +943,41 @@ export function judgePrompt(options) {
         "",
         `<how to invoke it>\n${options.invocation.trim()}\n</how to invoke it>`,
         "",
-        ...scripts.flatMap((script) => [`<conformance testing script path="${script.path}" run-as="${script.runs}">`, script.content.trimEnd(), "</conformance testing script>", ""]),
+        ...scripts.flatMap((script) => {
+            const tag = script.prepare === true ? "prepare script" : "conformance testing script";
+            return [`<${tag} path="${script.path}" run-as="${script.runs}">`, script.content.trimEnd(), `</${tag}>`, ""];
+        }),
+        ...(options.run.timedOut === true ? [...timedOutBlock(options.run), ""] : []),
         `<failing run exit="${options.run.exitCode}" command="${options.run.command}">`,
+        runTiming(options.run),
+        "",
         options.run.output.trimEnd(),
         "</failing run>",
         ...(options.escalation !== undefined ? ["", ...escalationBlock(options.escalation)] : []),
     ].join("\n");
 }
+/**
+ * For the judge, of a run the time limit stopped: what kept it from finishing - something that hangs, the code's or a
+ * test's, or a suite that is only slow, which no agent can give more time.
+ */
+export function timedOutBlock(run) {
+    const limit = Math.round(run.timeoutMs / 1000);
+    return [
+        "<timed out>",
+        `This run did not end on its own: the time limit stopped it at ${limit}s, and it counts as failed. Its output ends where it`,
+        "was stopped - the step running then, or the one after the last it printed, is what did not finish. Decide what kept it:",
+        "- code: the software hangs - it waits on input, keeps a server in the foreground, never exits, loops, or blocks on a",
+        "  lock or a call that never answers - when the tests reach it as the specification says.",
+        "- tests: a test waits for something that never comes - a process it started and never stops, output it waits for that",
+        "  the software never prints, a port or a file nobody makes, a sleep or a retry with no end.",
+        "- environment: nothing hangs - the output shows the run making steady progress right up to the limit (installing,",
+        "  building, test after test passing) and it needs more time than a test run may take. No agent can give it that: the",
+        "  run stops at once, and the person is told to raise the limit. Say from the output how far it got, and what took",
+        "  the time.",
+        "Judge from the output and the files, never from the time alone.",
+        "</timed out>",
+    ];
+}
 /** For the judge: the tests were fixed for this failure, and it did not go away - is it the code's? */
 function escalationBlock(escalation) {
     const changed = escalation.changed.length === 0 ? "changed no file" : `changed ${escalation.changed.slice(0, 8).join(", ")}${escalation.changed.length > 8 ? ` and ${escalation.changed.length - 8} more` : ""}`;
@@ -976,7 +1021,10 @@ export async function createCodeFixer(options) {
         model: options.model,
         label: `${options.label} coding agent`, role: "coding",
         system: `${fence.note}\n\n${workspaceBlock({ dir: options.buildDir, hasMyDiff: options.toolNames.includes("my_diff") })}`,
-        task: codeFixerTask(options),
+        // What the conformance testing script needs of the build, the code provides: the fixer reads it, not only a judge's quote of it.
+        task: options.testingScripts !== undefined
+            ? withTestingScripts(codeFixerTask(options), options.testingScripts.scripts, ["conformance", "unit"], options.testingScripts.root)
+            : codeFixerTask(options),
         compaction: { instructions: CODER_COMPACTION },
         ...(options.checkEnvironment !== undefined ? { checkEnvironment: options.checkEnvironment } : {}),
     });
@@ -1029,6 +1077,8 @@ export function codeFixPrompt(options) {
         `<why the code>\n${options.reason.trim()}\n</why the code>`,
         "",
         `<failing run command="${options.run.command}">`,
+        runTiming(options.run),
+        "",
         options.run.output.trimEnd(),
         "</failing run>",
         "",
@@ -1062,6 +1112,8 @@ export function escalatedFixPrompt(options) {
         "</evidence>",
         "",
         `<failing run command="${options.run.command}">`,
+        runTiming(options.run),
+        "",
         options.run.output.trimEnd(),
         "</failing run>",
         "",
@@ -1099,6 +1151,8 @@ export function testsFixPrompt(options) {
         `<why the tests>\n${options.reason.trim()}\n</why the tests>`,
         "",
         "<failing run>",
+        runTiming(options.run),
+        "",
         options.run.output.trimEnd(),
         "</failing run>",
         "",

package/dist/regenerate/conformance.js

@@ -9,12 +9,12 @@ import { abandonWork, diffSinceSnapshot, moduleDirOf, recordHistory, startWork,
 import { createWorkLedger, describeAccounts, SUBMIT_WORK_NOTE, unanswered, unansweredPrompt } from "./submit-work.js";
 import { CONFORMANCE_COMPACTION } from "./compaction.js";
 import { NOTES_TURN, notesForSession, passedOutcome, readNotes, reflectOnReview, withNotes } from "./notes.js";
-import { labelsPrompt, labelsPromptFor, misplacedConformanceTests, misplacedTestsPrompt, missingTestLabels } from "./test-kinds.js";
+import { acceptanceLabel, acceptanceText, labelsPrompt, labelsPromptFor, misplacedConformanceTests, misplacedTestsPrompt, missingTestLabels } from "./test-kinds.js";
 import { createScriptTool, findTestScripts, projectRootFor, requireTestScripts, runConformanceScript, runUnitScript, failureSignature, STUCK_ROUNDS, scriptFailure, staticChecksOnly, withTestingScripts, } from "./test-scripts.js";
 import { createWebTools, WEB_TOOLS } from "./web.js";
 import { createMyDiffTool } from "./self-check.js";
 import { createServiceTools } from "./services.js";
-import { missingToolchainError, toolchainError } from "./environment-error.js";
+import { missingToolchainError, timeLimitError, toolchainError } from "./environment-error.js";
 import { workspaceBlock } from "./workspace.js";
 import { codeFixPrompt, conflictKey, escalatedFixPrompt, createCodeFixer, answerAsExplorer, createAskExplorerTool, exploreInvocation, judgeFailure, SPEC_CONFLICT_RAISES, ENVIRONMENT_RAISES, environmentCheckPrompt, createTestReviewer, testReviewRequest, reviewFeedbackPrompt, testsFixPrompt, sortedRefs, } from "./conformance-agents.js";
 import { resolveRequiresChain } from "./requires.js";
@@ -157,9 +157,11 @@ export async function renderConformanceTests(options) {
     // may read the code but writes only tests. An explorer reads the code first and writes down how the
     // software is invoked, which is what the tests are written against.
     const label = `${moduleName} ${options.diff !== undefined ? "spec change" : frid}`;
-    const specContext = { moduleName, text: specText(specifications, moduleName, frids, required, options.change ?? options.diff, sectionModules(tree, moduleName)) };
     // For a spec change: the functionalities it touches, whose acceptance tests are labelled, checked and reviewed.
     const touched = forChange ? touchedFunctionalities(tree, frids, options.diff) : [];
+    // The acceptance tests the run's tests carry out: every functionality the change touches, or the one under test.
+    const carriedOut = forChange ? touched : [{ frid: frid, acceptanceTests: specifications[ACCEPTANCE_TESTS] ?? [] }];
+    const specContext = { moduleName, text: specText(specifications, moduleName, frids, required, carriedOut, options.change ?? options.diff, sectionModules(tree, moduleName)) };
     // Any functionality of the codebase - this module's, and those of the modules it requires. A
     // functionality's run may raise a spec conflict only with that functionality; a spec diff's, between any two.
     const own = (forChange ? frids : frids.slice(0, frids.indexOf(frid) + 1)).map((id) => ({ module: moduleName, frid: id }));
@@ -223,11 +225,15 @@ export async function renderConformanceTests(options) {
     };
     stage(`exploring the code for how it is invoked${options.model ? ` (${options.model})` : ""}`);
     // The explorer reads the conformance testing script too: it is how the tests are built and run, and so how they reach the software.
+    // The prepare script, where there is one, runs on the build before it - and only once it passes does the suite run.
     const testingScripts = [
+        ...(scripts.prepare !== undefined
+            ? [{ path: relative(root, scripts.prepare), runs: "<script> <copy of the build>, before every run of the conformance testing script", content: readFileSync(scripts.prepare, "utf8"), prepare: true }]
+            : []),
         { path: relative(root, scripts.conformance), runs: "<script> <copy of the build> <copy of the whole conformance suite>", content: readFileSync(scripts.conformance, "utf8") },
     ];
     // The acceptance tests the explorer's plan has to carry out, step by step.
-    const toCarryOut = (forChange ? touched : [{ frid: frid, acceptanceTests: specifications[ACCEPTANCE_TESTS] ?? [] }])
+    const toCarryOut = carriedOut
         .map((item) => ({ module: moduleName, frid: item.frid, count: item.acceptanceTests.length }))
         .filter((item) => item.count > 0);
     let invocation = await exploreInvocation({ buildDir, spec: specContext, scope, testingScripts, acceptance: toCarryOut, label, model: options.model, onDelta: options.onDelta });
@@ -319,6 +325,7 @@ export async function renderConformanceTests(options) {
             ],
             toolNames: ["run_unit_tests", "run_conformance_tests", ...fixerServices.names, "update_notes"],
             checkEnvironment: { run: () => runUnitScript(buildDir, scripts.unit), script: scripts.unit },
+            testingScripts: { scripts, root },
         });
         fixerConformanceRuns.newTurn();
         return ask(fixer, withNotes(text, buildDir), options.onDelta);
@@ -494,19 +501,6 @@ export async function renderConformanceTests(options) {
                         idleFixes = 0;
                     continue;
                 }
-                // A run out of time is not a question of test or code: the tests' environment is the conformance agent's.
-                if (recheck.timedOut === true) {
-                    const moved = await unmoved();
-                    if (moved === "stuck")
-                        break;
-                    if (moved === "asked")
-                        continue;
-                    stage("the conformance run did not get to the tests - back to the conformance agent");
-                    const fixed = await changing(() => askConformance(scriptFailure(recheck, "conformance testing"), options.onDelta));
-                    response = fixed.result;
-                    recordAttempt(rounds, recheck, `not a question of code or tests (${recheck.timedOut === true ? "the time limit" : `exit ${recheck.exitCode}`})`, { who: "conformance agent", files: fixed.files });
-                    continue;
-                }
                 stage(`conformance tests failing - working out whether the code, the tests or the spec are wrong${judgeModel ? ` (${judgeModel})` : ""}`);
                 // A fix that changed no file leaves the next run as this one: IDLE_FIXES of them in a row, and nothing will change.
                 const idle = (files) => {
@@ -541,6 +535,14 @@ export async function renderConformanceTests(options) {
                     onDelta: options.onDelta,
                 });
                 const reason = verdict.reason.split("\n")[0].slice(0, 160);
+                // Stopped by the time limit while it was still getting on: no agent can give it more time. The pulse stops now -
+                // rolled back, no retry - and the person is told how to raise the limit.
+                if (verdict.culprit === "environment" && recheck.timedOut === true) {
+                    stage(`the conformance run needs more than the ${Math.round(recheck.timeoutMs / 1000)}s a test run may take - stopping`);
+                    if (start !== undefined)
+                        abandonWork(start, moduleName, "the conformance testing script needs more time than a test run may take");
+                    throw timeLimitError(recheck, { moduleName, script: scripts.conformance, kind: "conformance", reason: verdict.reason });
+                }
                 // The environment - what the run is given - is the person's to fix: once judged so twice in a row, the run stops.
                 if (verdict.culprit === "environment") {
                     environmentRaises += 1;
@@ -692,7 +694,7 @@ export async function renderConformanceTests(options) {
                     buildDir,
                     conformanceDir,
                     spec: specContext,
-                    functionalities: forChange ? touched : [{ frid: frid, acceptanceTests: specifications[ACCEPTANCE_TESTS] ?? [] }],
+                    functionalities: carriedOut,
                     forChange,
                     invocation,
                     label,
@@ -952,9 +954,10 @@ export function failureLine(run) {
 }
 /**
  * The spec as the explorer, the judge and the coding agent are shown it - its functional specs
- * numbered with their ids, so a spec conflict can name them.
+ * numbered with their ids, so a spec conflict can name them, and the acceptance tests the run carries
+ * out labelled as their tests are, so a verdict can quote them.
  */
-function specText(specifications, moduleName, frids, required, diff, modules) {
+function specText(specifications, moduleName, frids, required, acceptance, diff, modules) {
     // Every functionality in the codebase, each with its module: the required modules' first, then this one's.
     const numbered = (key, items) => key === FUNCTIONAL_REQUIREMENTS
         ? [
@@ -962,10 +965,14 @@ function specText(specifications, moduleName, frids, required, diff, modules) {
         ]
         : items;
     // Each section in its tag, as every agent is shown it (see specPrimer).
-    const sections = [DEFINITIONS, NON_FUNCTIONAL_REQUIREMENTS, TEST_REQUIREMENTS, FUNCTIONAL_REQUIREMENTS, ACCEPTANCE_TESTS]
+    const sections = [DEFINITIONS, NON_FUNCTIONAL_REQUIREMENTS, TEST_REQUIREMENTS, FUNCTIONAL_REQUIREMENTS]
         .map((key) => [key, specifications[key] ?? []])
         .filter(([, items]) => items.length > 0)
-        .map(([key, items]) => (key === FUNCTIONAL_REQUIREMENTS || key === ACCEPTANCE_TESTS ? section(key, numbered(key, items)) : moduleSection(key, items, moduleName, modules)));
+        .map(([key, items]) => (key === FUNCTIONAL_REQUIREMENTS ? section(key, numbered(key, items)) : moduleSection(key, items, moduleName, modules)));
+    // Not only the last functionality's: a spec change's run carries out the acceptance tests of every one it touches.
+    const tests = acceptance.flatMap((item) => item.acceptanceTests.map((test, index) => `${acceptanceLabel(item.frid, index + 1)}: ${acceptanceText(test)}`));
+    if (tests.length > 0)
+        sections.push(section(ACCEPTANCE_TESTS, tests));
     if (diff !== undefined)
         sections.push(specChangeBlock(diff));
     return sections.join("\n\n");

package/dist/regenerate/debugger.js

@@ -4,6 +4,7 @@ import { Type } from "typebox";
 import { ask, BEGIN, createExecutor } from "../pi.js";
 import { confinedTools } from "./confine.js";
 import { createWebTools, WEB_TOOLS } from "./web.js";
+import { runTiming } from "./test-scripts.js";
 /**
  * A fresh look at a failure the coding agent keeps hitting.
  *
@@ -24,9 +25,12 @@ export function diagnosisText(diagnosis) {
 }
 function debugPrompt(input) {
     const script = input.script !== undefined && existsSync(input.script) ? readFileSync(input.script, "utf8") : undefined;
+    const timedOut = input.run?.timedOut === true;
     return [
-        `The unit tests of the codebase in ${input.buildDir} have failed the same way ${input.times} times in a row. The coding agent`,
-        "keeps changing things and the failure does not move: it has not seen the cause. You look at it with fresh eyes.",
+        timedOut
+            ? `The unit testing script of the codebase in ${input.buildDir} did not finish: the time limit stopped it. You look at it with fresh eyes.`
+            : `The unit tests of the codebase in ${input.buildDir} have failed the same way ${input.times} times in a row. The coding agent`,
+        ...(timedOut ? [] : ["keeps changing things and the failure does not move: it has not seen the cause. You look at it with fresh eyes."]),
         "",
         "Find the root cause - the one thing that makes this failure happen - and the change that fixes it. Read the failing",
         "run below, the tests it names, the code they exercise, the build files (pyproject.toml, package.json, go.mod, pom.xml,",
@@ -37,6 +41,21 @@ function debugPrompt(input) {
         "",
         "You change nothing - you only read. When you know, call submit_diagnosis: the root cause, with the evidence for it,",
         "and the fix - which files, and exactly what to change in them.",
+        ...(timedOut
+            ? [
+                "",
+                "<timed out>",
+                "The run's output ends where the time limit stopped it: the test running then, or the step after the last one it",
+                "printed, is what did not finish. Find what kept it:",
+                "- something hangs - a test or the code waits on input, keeps a server in the foreground, never exits, loops, blocks",
+                "  on a lock or a network call that never answers: the root cause and the fix, as for any failure;",
+                "- nothing hangs - the output shows the run making steady progress right up to the limit (installing, building, test",
+                "  after test passing) and it needs more time than a test run may take: set too_slow, and say in root_cause from the",
+                "  output how far it got and what took the time. No agent can give it more time: the work stops, and the person is",
+                "  told to raise the limit. Judge from the output and the files, never from the time alone.",
+                "</timed out>",
+            ]
+            : []),
         "",
         "<what the change is meant to do>",
         input.task.trim(),
@@ -44,6 +63,7 @@ function debugPrompt(input) {
         ...(script !== undefined ? ["", `<testing script path="${input.script}">`, script.trimEnd(), "</testing script>"] : []),
         "",
         "<failing run>",
+        ...(input.run !== undefined ? [runTiming(input.run), ""] : []),
         input.failure.trimEnd().slice(-8000),
         "</failing run>",
     ].join("\n");
@@ -58,10 +78,14 @@ export async function debugFailure(input) {
         parameters: Type.Object({
             root_cause: Type.String({ description: "The one thing that makes the failure happen, and the evidence for it: the lines of the run, the code, the docs" }),
             fix: Type.String({ description: "The change that fixes it: which files, and exactly what to change in them" }),
+            too_slow: Type.Optional(Type.Boolean({
+                description: "Only for a run the time limit stopped: true when nothing hangs - it was making progress up to the limit and needs more time than a test run may take",
+            })),
         }),
         async execute(_id, params) {
             const given = params;
-            found = { rootCause: given.root_cause, fix: given.fix };
+            // Too slow is a verdict on a run the limit stopped, and on no other.
+            found = { rootCause: given.root_cause, fix: given.fix, ...(given.too_slow === true && input.run?.timedOut === true ? { tooSlow: true } : {}) };
             return { content: [{ type: "text", text: "Recorded." }], details: {}, terminate: true };
         },
     });

package/dist/regenerate/environment-error.js

@@ -224,6 +224,23 @@ export function toolchainError(missing, details) {
         source: "harness",
     });
 }
+/**
+ * A testing script the time limit stopped while it was still making progress - judged so by the judge or the debugger,
+ * from where its output stops: nothing hangs, the suite needs more time than a test run may take. No agent can give it
+ * that, so the pulse stops and the person is told how to.
+ */
+export function timeLimitError(run, details) {
+    const limit = Math.round(run.timeoutMs / 1000);
+    return new EnvironmentError({
+        moduleName: details.moduleName,
+        missing: `more time for a test run than its ${limit}s limit`,
+        neededBy: `the ${details.kind} testing script, \`${basename(details.script)}\``,
+        shownBy: `it ran ${Math.round((run.elapsedMs ?? run.timeoutMs) / 1000)}s and was stopped at the limit, still making progress - ${details.reason.replace(/\s+/g, " ").trim().slice(0, 400)}`,
+        fix: `give a test run more time - set TURRI_COMMAND_TIMEOUT, in seconds, above ${limit} (e.g. TURRI_COMMAND_TIMEOUT=${limit * 3}) in the ` +
+            "environment turri runs in - or make the suite faster: what it installs or builds on every run, and the tests that take longest.",
+        source: "agent",
+    });
+}
 /** The harness's run of a testing script, when it could not find its toolchain: the error that stops the pulse. */
 export function missingToolchainError(run, details) {
     const missing = missingToolchain(run);

package/dist/regenerate/prompt.js

@@ -278,8 +278,10 @@ function itemText(serialized) {
         return item;
     const { markdown, [ACCEPTANCE_TESTS]: tests } = (item ?? {});
     const lines = [markdown ?? JSON.stringify(item)];
+    // Each test is an item of the tree, its text in its markdown - as spec-steps.ts `itemText` reads it.
+    const testText = (test) => (typeof test === "string" ? test : (test?.markdown ?? JSON.stringify(test)));
     if (Array.isArray(tests) && tests.length > 0)
-        lines.push("  <acceptance tests>", ...tests.map((test) => `  ${String(test)}`), "  </acceptance tests>");
+        lines.push("  <acceptance tests>", ...tests.map((test) => `  ${testText(test).replace(/\n/g, "\n  ")}`), "  </acceptance tests>");
     return lines.join("\n");
 }
 const indent = (text, by) => text.split("\n").map((line) => `${by}${line}`).join("\n");

package/dist/regenerate/regenerate.js

@@ -20,7 +20,7 @@ import { createWebTools, WEB_TOOLS } from "./web.js";
 import { createCanItFailTool, createMyDiffTool, selfCheckNote } from "./self-check.js";
 import { createServiceTools } from "./services.js";
 import { createUnitGate } from "./unit-gate.js";
-import { toolchainError } from "./environment-error.js";
+import { timeLimitError, toolchainError } from "./environment-error.js";
 import { workspaceBlock } from "./workspace.js";
 import { ATTEMPTS, describeStop } from "./stop-report.js";
 import { summarizeDiff } from "./job-panel.js";
@@ -333,6 +333,8 @@ export async function renderSpecDiff(options) {
     let draftAsks = 0;
     /** What the unit testing script needs and does not find, last run. */
     let missing;
+    /** The unit testing script needs more time than a test run may take - the debugger's finding: the pulse stops on it. */
+    let slow;
     /** The last round failed on the script's static checks alone: the unit tests passed. */
     let staticOnly = false;
     const testRemovals = [];
@@ -423,24 +425,35 @@ export async function renderSpecDiff(options) {
                         seenFailures.set(key, seen);
                     let prompt = verdict.prompt;
                     const circling = loop.stuck;
-                    if (circling || (key !== undefined && seen >= STUCK_ROUNDS)) {
+                    // A run the time limit stopped is looked at at once: whether something hangs, or the suite only needs more time.
+                    const timedOut = verdict.run.timedOut === true;
+                    if (circling || timedOut || (key !== undefined && seen >= STUCK_ROUNDS)) {
                         if (key !== undefined)
                             seenFailures.delete(key);
                         const times = circling ? Math.max(seen, loop.failing) : seen;
                         loop.reset();
                         stage(circling
                             ? `the coding agent is going round in circles - ${times} failing runs - a debugger looks at it with fresh eyes`
-                            : `the same failure ${seen} times - a debugger looks at it with fresh eyes`);
+                            : timedOut && seen < STUCK_ROUNDS
+                                ? `the unit testing script did not finish in ${Math.round(verdict.run.timeoutMs / 1000)}s - a debugger works out whether something hangs`
+                                : `the same failure ${seen} times - a debugger looks at it with fresh eyes`);
                         const found = await debugFailure({
                             buildDir,
                             failure: verdict.run.output,
                             times,
                             task: steeredChange() ?? codingText ?? changeText ?? options.diff ?? "",
                             script: verdict.script,
+                            run: verdict.run,
                             label: `${moduleName} spec change`,
                             ...(debuggerModel !== undefined ? { model: debuggerModel } : {}),
                             onDelta: options.onDelta,
                         }).catch(() => undefined);
+                        // Nothing hangs: the suite needs more time than a test run may take, and no agent can give it that.
+                        if (found?.tooSlow === true) {
+                            stage(`the unit testing script needs more than the ${Math.round(verdict.run.timeoutMs / 1000)}s a test run may take - stopping`);
+                            slow = timeLimitError(verdict.run, { moduleName, script: verdict.script, kind: "unit", reason: found.rootCause });
+                            return false;
+                        }
                         if (found !== undefined) {
                             const cause = found.rootCause.split("\n")[0].slice(0, 240);
                             reportActivity(`  the debugger: ${cause.slice(0, 160)}`);
@@ -486,6 +499,7 @@ export async function renderSpecDiff(options) {
                 ...(options.cases !== undefined ? { cases: options.cases } : {}),
                 label: `${moduleName} spec change`,
                 ...(scripts.runOn === "copy" ? { unitScript: scripts.unit } : {}),
+                testingScript: { path: relative(root, scripts.unit), content: readFileSync(scripts.unit, "utf8") },
                 model: options.judgeModel ?? modelSpec(inheritModel(configForSpec(tip.plainFile).judgeModel, modelOnly(options.model))),
                 send: async (next) => {
                     response = await ask(executor, next, options.onDelta);
@@ -543,13 +557,17 @@ export async function renderSpecDiff(options) {
         stage(unitReviewStopped ? "the unit tests pass but still leave the change untested - undoing the change" : "unit tests NOT passing - undoing the change");
         abandonWork(unit, moduleName, missing !== undefined
             ? "the unit testing script could not run"
-            : staticOnly
-                ? "the spec change did not pass its static checks"
-                : "the spec change did not pass its unit tests");
+            : slow !== undefined
+                ? "the unit testing script needs more time than a test run may take"
+                : staticOnly
+                    ? "the spec change did not pass its static checks"
+                    : "the spec change did not pass its unit tests");
         // The environment lacks what the testing script needs: the pulse stops, rolled back, and the person is told.
         if (missing !== undefined) {
             throw toolchainError(missing, { moduleName, script: scripts.unit, kind: "unit" });
         }
+        if (slow !== undefined)
+            throw slow;
         return { ...base, passed: false, stoppedAt: "unit tests", ...(staticOnly && !unitReviewStopped ? { staticChecks: true } : {}) };
     }
     stage("unit tests pass - landing the change on them");

package/dist/regenerate/test-scripts.js

@@ -335,7 +335,15 @@ async function runScript(cwd, script, args, logDir, options = {}) {
     const noted = withScriptLineNote(text, script);
     // Its failures, counts and end are what the agent is shown; the whole is kept, a read_output away.
     const log = keepTestOutput(logDir, basename(script), noted);
-    return { command: shown(script), exitCode: timedOut === true ? 124 : exitCode, output: shownTestOutput(noted, log), timedOut: timedOut === true, timeoutMs, ...(log !== undefined ? { log } : {}) };
+    return {
+        command: shown(script),
+        exitCode: timedOut === true ? 124 : exitCode,
+        output: shownTestOutput(noted, log),
+        timedOut: timedOut === true,
+        timeoutMs,
+        elapsedMs: Date.now() - started,
+        ...(log !== undefined ? { log } : {}),
+    };
 }
 /** What bash says when a line of the script itself cannot run: a command or file that is not there, or not runnable. */
 const OWN_LINE_FAILURE = /: (No such file or directory|command not found|Permission denied)$/;
@@ -589,6 +597,19 @@ function loopNotice(loop) {
         "the failure with fresh eyes, and you will be given what it finds.",
     ].join("\n");
 }
+/**
+ * How long a run took against its time limit, in a line: for whoever judges it or works out its cause - so a run that
+ * ended on its own is never taken for one the limit stopped, nor the other way.
+ */
+export function runTiming(run) {
+    const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`;
+    const limit = seconds(run.timeoutMs);
+    if (run.timedOut === true) {
+        return `The time limit stopped this run: it had run ${seconds(run.elapsedMs ?? run.timeoutMs)} of the ${limit} a test run may take, and had not finished.`;
+    }
+    const took = run.elapsedMs !== undefined ? `after ${seconds(run.elapsedMs)} of the ${limit} a test run may take` : `well within the ${limit} a test run may take`;
+    return `This run ended on its own - exit ${run.exitCode}, ${took}. The time limit did not stop it and played no part in how it ended.`;
+}
 /** A run the time limit stopped: failed, and what most often keeps a test run from ending. */
 function timedOutFailure(run, what) {
     const limit = Math.round(run.timeoutMs / 1000);

package/dist/regenerate/unit-review.js

@@ -63,6 +63,17 @@ export function unitReviewTask(options) {
         options.spec.trim(),
         "</specification>",
         "",
+        ...(options.testingScript !== undefined
+            ? [
+                "The harness runs the unit tests with this script, and it decides whether they pass. Read it for which tests it",
+                "runs and how: a test it never reaches - a file it does not pick up, a folder it does not look in - is not run, and",
+                "checks nothing.",
+                `<unit testing script path="${options.testingScript.path}">`,
+                options.testingScript.content.trimEnd(),
+                "</unit testing script>",
+                "",
+            ]
+            : []),
         ...(options.cases !== undefined ? [...codeReviewCasesBlock(options.cases), ""] : []),
         // A job that touches no functional spec is held only to that: the checks of behaviour are not its.
         ...(mechanical ? [] : ["Check that the code does what the specification states for this work:"]),

package/docs/regenerating.md

@@ -160,6 +160,10 @@ runs the unit testing script itself**, and only that run counts (see
   tried. An agent circles a cause it has not thought of - an in-memory database every connection opens
   empty, a test runner that leaves the code off its path - and this is the way out. The rounds go on as
   before, up to 20; should the same failure come round three times again, the debugger looks again.
+  **A run the time limit stopped** is looked at by the debugger at once, told how long the run had and that the limit
+  stopped it: it finds what hangs - the root cause and the fix, as above - or that nothing does, the run getting on up
+  to the limit and needing more time than a test run may take. Then the pulse stops, rolled back and not retried, and
+  the person is told to raise `TURRI_COMMAND_TIMEOUT` or make the suite faster.
 - **It goes round in circles in its own runs.** An agent stuck on a failure can try the same few changes in
   turn - three signatures of a stand-in function, say - and run the tests after each, dozens of times in one
   turn. Its own `run_unit_tests` runs are watched (`createLoopWatch` in `test-scripts.ts`): the code folder back
@@ -262,7 +266,8 @@ Passing unit tests prove only what they check, and a coding agent is quick to wr
 next to nothing - that a function exists, that a result is not empty, that a status is 200 - or code
 that no test reaches. So, with `review: true` in the [config](config.md), once the unit testing script
 passes a reviewer on the judge model reads what the work wrote, beside the spec, and changes nothing. It
-reviews both halves. The review is off by default - `review: false`, or the key left out - and so is the
+reviews both halves, with the unit testing script word for word in its prompt: a test the script never reaches is
+not run, and checks nothing. The review is off by default - `review: false`, or the key left out - and so is the
 conformance tests' review: the work goes on once its tests pass.
 
 - **The code.** Every behaviour, edge case, validation rule and error the spec states for this work
@@ -499,8 +504,9 @@ testing script, stops before it starts and says which skill adds it.
 `prepare-environment-script:` in the config, else the one `prepare_environment*.sh` at the root, else in
 `test_scripts/` - on the build, before every run of the conformance testing script: the harness's own, and the agents'
 `run_conformance_tests`. Both get the same build folder - the code in place, or the same fresh copy - so what it sets up
-for that build, the conformance testing script finds. With conformance off it is not run. The conformance agents are
-shown it, before the conformance testing script.
+for that build, the conformance testing script finds. With conformance off it is not run. Every agent shown the
+conformance testing script - the explorer, the conformance agent, the judge, the code fixer and the repair agent - is
+shown it too, before the conformance testing script.
 
 **Where they run.** On this machine, as you would run them: unfenced - no sandbox, no command check, no audit - in
 your own environment with this machine's toolchains, turri's model providers' API keys alone left out. They are yours,
@@ -832,7 +838,8 @@ coding agent that fixes the code is told the same of the tests: it can read them
 Before the test author starts, an explorer - an agent in
 the code's folder with the file tools alone: it reads the code, runs nothing and changes nothing - hands over everything
 the author needs, in parts: how to build and run it; how the conformance tests reach it as the project's
-conformance testing script runs them - the explorer is given the script, and says what it builds or
+conformance testing script runs them - the explorer is given the script, and the prepare script before it where there
+is one, and says what they build or
 installs and what the tests depend on or start (a package, a module, an entry point, by its exact name);
 how each functionality is reached from outside
 (the command, or the route, method and the request and response fields); what it needs around it; and
@@ -915,7 +922,8 @@ culprit: what it does with the tests, the tests fit; what it needs of the build,
 - **the code:** the coding agent - in the code's folder, reading the tests but never changing them - fixes it,
   and keeps its unit tests passing. The judge quotes the spec's own words the code does not meet, or the testing
   script's for what it needs of the build (`script_needs`: a build output, an entry point, a module) - each checked
-  by the harness against the text it quotes;
+  by the harness against the text it quotes. The acceptance tests are in that text, labelled as their tests are:
+  for a spec change, those of every functional spec it touches, not only the last one's;
 - **the tests:** the conformance agent corrects them to what the spec says;
 - **the spec:** two functional specs cannot both hold, whatever the code does;
 - **the environment:** neither the code nor the tests can fix it - only the person can, in what the run is given:
@@ -957,7 +965,9 @@ spending the rest of its 20 attempts on it. Two failures are the same once what
 taken out of their lines - numbers, times, paths, colour codes, and the ids a service makes up for each
 response, a UUID or a correlation id. Nothing changes either when the agent sent to fix the code or the tests
 changes no file: two such fixes in a row, and the run stops. The conformance agent has the testing script in its prompt,
-and is told to fit the tests' own build to it.
+and is told to fit the tests' own build to it. The code fixer has the testing scripts its fixes are run by - the
+conformance testing script, the prepare script before it, and the unit testing script - word for word in its task:
+what the conformance testing script needs of the build, it reads there, not only in the judge's quote of it.
 
 **Escalation.** A failure judged the tests' goes to the conformance agent first: it owns the tests. When the run then
 fails the same way - the fix changed nothing, or nothing that mattered - the judge is asked again, now whether the
@@ -978,9 +988,18 @@ the run failed with, how it was judged, and which files the agent sent to fix it
 attempt's line says what came of it - and each judge reads them first, so it does not send a
 failure the way that already failed. A run that does not pass is undone whole, its notes with it.
 
-Then the suite runs again. A run stopped by its time limit goes back to the conformance agent without a judge: the
-tests' environment is its. So does a run that exits 80 - the tests pass and only their static checks do not (see
-[the static checks](#the-static-checks)): the findings are the tests'. A run whose script cannot find its
+Then the suite runs again. A run that exits 80 goes back to the conformance agent without a judge - the tests pass and
+only their static checks do not (see [the static checks](#the-static-checks)): the findings are the tests'.
+
+**Timing.** Every run is shown to the judge, the debugger and whoever fixes it with how long it ran against its time
+limit (`runTiming` in `test-scripts.ts`): `This run ended on its own - exit 1, after 37.2s of the 120.0s a test run
+may take`, or `The time limit stopped this run: it had run 120.0s of the 120.0s ...` - so none takes a run that
+ended on its own for one the limit stopped, or the other way. A run the time limit stopped is judged like any other,
+and the judge is told what keeps a run from finishing: the code hangs (it waits on input, keeps a server in the
+foreground, never exits) - the code's; a test waits for what never comes - the tests'; or nothing hangs, the output
+showing the run getting on up to the limit, and it needs more time than a test run may take - the environment. That
+verdict stops the pulse at once, rolled back and not retried, and the person is told to raise `TURRI_COMMAND_TIMEOUT`
+or make the suite faster - no agent can give a run more time. A run whose script cannot find its
 toolchain stops the pulse instead - see [when the environment lacks something](#when-the-environment-lacks-something).
 Unit tests failing after the suite passes go to the coding agent. It goes round up to 20 times; still failing, the work is undone.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@turri-dev/cli",
-  "version": "0.1.3-next.a5d3797",
+  "version": "0.1.3-next.b675516",
   "license": "MIT",
   "type": "module",
   "repository": {
@@ -33,8 +33,8 @@
     "grammar:sync": "node scripts/sync-plyn-grammar.mjs"
   },
   "dependencies": {
-    "@earendil-works/pi-coding-agent": "^1.1.0",
-    "@earendil-works/pi-tui": "^1.1.0",
+    "@earendil-works/pi-coding-agent": "^1.2.0",
+    "@earendil-works/pi-tui": "^1.2.0",
     "@langchain/langgraph": "1.4.15",
     "gray-matter": "^4.0.3",
     "js-yaml": "^5.4.2",