@turri-dev/cli 0.1.3-next.e5599c1 → 0.1.3

package/dist/config.js

@@ -189,8 +189,9 @@ export const ROLE_THINKING = {
     diagnosis: "medium",
 };
 /** The roles on the conversation's side: without their own entry they take the chat's model, not `default`. */
-// The release is not: it follows steps - a script, a start command - and runs on the coding model, thinking low.
-const CHAT_SIDE = ["chat", "chooser", "planner", "diagnosis"];
+// The release is not: it follows steps - a script, a start command - and runs on the coding model, thinking low. Nor
+// is the diagnosis: a post-mortem of a job that failed, read by the person - it runs on the coding model, thinking medium.
+const CHAT_SIDE = ["chat", "chooser", "planner"];
 function isThinkingLevel(value) {
     return typeof value === "string" && THINKING_LEVELS.includes(value);
 }

package/dist/extension.js

@@ -9,7 +9,7 @@ import { allocateImageId, Box, getCapabilities, getPngDimensions, isKeyRelease,
 import { defineTool } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
 import { CODING_MODEL_ENV } from "./chat-models.js";
-import { AGENT_ROLES, CONFIG_FILE, configForSpec, hasProjectConfig, LEGACY_CONFIG_FILE, projectConfigFile, stackConfigFileIn, stackOf, stacksOf, withStack, modelSpec, projectModels, readProjectConfig, roleModel, splitModelSpec } from "./config.js";
+import { AGENT_ROLES, CONFIG_FILE, configForSpec, hasProjectConfig, LEGACY_CONFIG_FILE, projectConfigFile, stackConfigFileIn, stackOf, stacksOf, withStack, modelOnly, modelSpec, projectModels, readProjectConfig, roleModel, splitModelSpec } from "./config.js";
 import { skillRunDir } from "./regenerate/paths.js";
 import { sweepOrphanedGroups } from "./regenerate/shell.js";
 import { readScript, scriptAgainstMachine, stackOnMachine, toolchainsBlock } from "./stack.js";
@@ -90,15 +90,23 @@ aside) {
             // The question, with what the asker gives to read for it - a brief to confirm - above it; what it said of its work
             // before asking is kept in its record, never put in the dialog.
             // A dialog of turri's own: a long brief scrolls above the question, and the options always show (question-dialog.ts).
-            const dialog = (question, context, options) => ctx.ui.custom((tui, theme, _keybindings, done) => new QuestionDialog({
-                question,
-                ...(context !== undefined ? { context } : {}),
-                ...(options !== undefined ? { options } : {}),
-                rows: () => tui.terminal.rows,
-                style: { accent: (text) => theme.fg("accent", text), muted: (text) => theme.fg("muted", text), bold: (text) => theme.bold(text) },
-                done,
-                requestRender: () => tui.requestRender(),
-            }));
+            const dialog = (question, context, options) => ctx.ui.custom((tui, theme, _keybindings, done) => {
+                const shown = new QuestionDialog({
+                    question,
+                    ...(context !== undefined ? { context } : {}),
+                    ...(options !== undefined ? { options } : {}),
+                    rows: () => tui.terminal.rows,
+                    style: { accent: (text) => theme.fg("accent", text), muted: (text) => theme.fg("muted", text), bold: (text) => theme.bold(text) },
+                    done: (answer) => {
+                        if (questionShown === shown)
+                            questionShown = undefined;
+                        done(answer);
+                    },
+                    requestRender: () => tui.requestRender(),
+                });
+                questionShown = shown;
+                return shown;
+            });
             const asking = (asked) => runAsk(asked, {
                 select: (title, options, context) => dialog(title, context, options),
                 input: (title, context) => dialog(title, context),
@@ -117,6 +125,12 @@ aside) {
 }
 /** The side panel, as a question open in the chat reaches it: set once the extension has its panel. */
 let questionPeek;
+/**
+ * The question open in the chat now, if any. The side panel gives the keys back to it: pi would give them to what had the
+ * focus when the panel first opened, and a panel kept beside the chat outlives that question - the keys would go to a
+ * dialog already answered, and nothing would take them.
+ */
+let questionShown;
 /** A widget with nothing to draw: below the editor, it takes no line. */
 const PEEK_KEY_WIDGET = "turri-peek-key";
 /**
@@ -186,7 +200,8 @@ export default function (pi) {
                 projectDir: cwd,
                 harnessDir: PACKAGE_ROOT,
                 jobLog,
-                model: modelSpec(projectModels(readProjectConfig(cwd)).harness),
+                // The coding model the job ran on - its model alone: the diagnosis thinks as its role has it.
+                model: modelSpec(modelOnly(job.model ?? modelSpec(projectModels(readProjectConfig(cwd)).coding))),
             });
             return diagnosis !== undefined ? describeDiagnosis(diagnosis) : undefined;
         },
@@ -1507,6 +1522,7 @@ function registerJobTools(pi, jobs, cwd) {
         }
         needsLooking.delete(jobs);
         if (panel !== undefined) {
+            panel.keepChatFocus();
             panel.handle?.setHidden(false);
             panel.keys = true;
             panel.handle?.focus();
@@ -1514,7 +1530,21 @@ function registerJobTools(pi, jobs, cwd) {
         }
         // Not awaited: the panel lives on while the jobs it shows run.
         void ctx.ui.custom((tui, theme, _keybindings, done) => {
-            const close = () => done(undefined);
+            // What the chat shows when the panel takes the keys - its editor, or a pi dialog - for when it gives them back.
+            const focusedNow = () => tui.getFocusedComponent?.() ?? null;
+            let chatFocus = focusedNow();
+            const keepChatFocus = () => {
+                const focused = focusedNow();
+                if (focused !== null && focused !== component && !(focused instanceof QuestionDialog))
+                    chatFocus = focused;
+            };
+            // The keys back to the chat: to the question open there now, else to what it showed - never to a question answered
+            // while the panel was open.
+            const giveBack = () => entry.handle?.unfocus({ target: questionShown ?? chatFocus });
+            const close = () => {
+                giveBack();
+                done(undefined);
+            };
             panelTerminal = tui.terminal;
             let component;
             // Beside the chat: the chat drawn narrower, on the left, and a click on either side giving it the keys - undone
@@ -1547,10 +1577,11 @@ function registerJobTools(pi, jobs, cwd) {
             // Beside the chat, back to it: the chat has the keys, the panel stays.
             const toChat = () => {
                 entry.keys = false;
-                entry.handle?.unfocus();
+                giveBack();
                 tui.requestRender();
             };
             const toPanel = () => {
+                keepChatFocus();
                 entry.keys = true;
                 entry.handle?.focus();
                 tui.requestRender();
@@ -1603,7 +1634,7 @@ function registerJobTools(pi, jobs, cwd) {
                     return focused !== null && focused !== component;
                 },
             });
-            const entry = { view, close, keys: true };
+            const entry = { view, close, keepChatFocus, keys: true };
             panel = entry;
             layOut();
             // Every key goes to the panel while it shows, whatever has the focus: a question the chat model asks in the

package/dist/loop-cli.js

@@ -69,7 +69,7 @@ async function runLoopReporting(options) {
     const registry = createJobRegistry({
         retries: 1,
         diagnose: async (job, failures, jobLog) => {
-            const diagnosis = await diagnoseFailure({ job, failures, projectDir, harnessDir: PACKAGE_ROOT, jobLog, model: modelSpec(projectModels(project).harness) ?? model });
+            const diagnosis = await diagnoseFailure({ job, failures, projectDir, harnessDir: PACKAGE_ROOT, jobLog, model: modelSpec(modelOnly(job.model ?? model)) });
             return diagnosis !== undefined ? describeDiagnosis(diagnosis) : undefined;
         },
         onChange: (job) => {

package/dist/pi-launch.js

@@ -11,6 +11,7 @@ import { fileURLToPath, pathToFileURL } from "node:url";
 import { main, SettingsManager } from "@earendil-works/pi-coding-agent";
 import { ProcessTerminal } from "@earendil-works/pi-tui";
 import "./terminal-images.js";
+import { resumeLine } from "./resume-line.js";
 /**
  * pi, started the way its own CLI starts it, for `turri`.
  *
@@ -31,6 +32,10 @@ const setTitle = ProcessTerminal.prototype.setTitle;
 ProcessTerminal.prototype.setTitle = function (title) {
     setTitle.call(this, title.replace(/^π(?= |$)/, "🪼 turri"));
 };
+// What pi prints on the way out to resume the session names pi - its APP_NAME, a constant - which may not be on the
+// PATH at all: turri takes the same flags, so it is turri the line names.
+const write = process.stdout.write.bind(process.stdout);
+process.stdout.write = ((chunk, ...rest) => write(typeof chunk === "string" ? resumeLine(chunk) : chunk, ...rest));
 // pi's own keys stand: a new line is shift+enter or ctrl+j. turri's side panel is on ctrl+h (job-panel.ts PANEL_KEY).
 // What pi's CLI does before main: the process title, pi's environment markers, the HTTP
 // dispatcher. Not exported, so loaded from the file - beside pi's entry point, wherever npm put it.

package/dist/question-dialog.js

@@ -58,18 +58,23 @@ export class QuestionDialog {
         const choices = this.options.options ?? [];
         const body = this.input !== undefined
             ? this.input.render(inner)
-            : choices.map((choice, index) => (index === this.selected ? style.accent(`› ${choice}`) : `  ${choice}`));
+            : // A long option wraps, its lines under its first, rather than cut off where the person cannot read it.
+                choices.flatMap((choice, index) => wrapTextWithAnsi(choice, Math.max(1, inner - 2)).map((line, at) => {
+                    const shown = `${at === 0 && index === this.selected ? "›" : " "} ${line}`;
+                    return index === this.selected ? style.accent(shown) : shown;
+                }));
         const hint = this.input !== undefined
             ? "enter answers · esc dismisses"
             : "↑↓ choose · enter picks · esc dismisses";
         // The brief gets what the question, the options and the hints leave of the screen.
-        const fixed = question.length + body.length + 4;
+        const fixed = question.length + body.length + 5;
         const room = Math.max(3, this.options.rows() - KEPT_ABOVE - fixed);
         const scrolls = brief.length > room;
         this.window = scrolls ? room - 1 : brief.length;
         this.maxScroll = scrolls ? brief.length - this.window : 0;
         this.scroll = Math.min(this.scroll, this.maxScroll);
-        const lines = [];
+        // Set apart from what is drawn above it - the job panel's hints.
+        const lines = [""];
         if (brief.length > 0) {
             lines.push(...brief.slice(this.scroll, this.scroll + this.window).map(pad));
             if (scrolls) {

package/dist/questioning.js

@@ -43,7 +43,7 @@ export function questioningQuestion(phases) {
     return {
         question: `How many questions should I ask you in ${named}?`,
         options: QUESTIONING_LEVELS.map((level) => level.option),
-        context: `Light, medium and thorough cap each of ${named} at 3, 5 and 10 questions - broader with fewer, so every level ` +
+        context: `Light, medium and thorough cap ${phases.length === 1 ? named : `each of ${named}`} at 3, 5 and 10 questions - broader with fewer, so every level ` +
             "covers the same ground, only in less or more depth. What the questions leave open, turri decides itself and " +
             "says what it chose. With best judgement, turri asks as much as it judges each phase needs. Each phase still " +
             "ends with your confirmation of its summary.",

package/dist/regenerate/activity.js

@@ -183,6 +183,8 @@ export function describeToolCall(name, args) {
             return `± my_diff`;
         case "update_notes":
             return `✐ update_notes`;
+        case "report_spec_gap":
+            return `? spec gap: ${text("question") ?? ""}`.trimEnd();
         case "copy_resource":
             return `⎘ copy_resource ${path ?? text("to") ?? ""}`.trimEnd();
         case "can_it_fail":

package/dist/regenerate/conform.js

@@ -7,7 +7,7 @@ import { abandonWork, startWork } from "./lineage.js";
 import { ensureRegressionPasses } from "./regression.js";
 import { modulesRoot, projectDirOf } from "./paths.js";
 import { applySpecDiff, snapshotSpec, specGapsText } from "./regenerate.js";
-import { linesAddedSince, readNotes } from "./notes.js";
+import { gapMark, gapsSince } from "./notes.js";
 import { sectionsOf } from "./state.js";
 import { describeStop } from "./stop-report.js";
 import { missingTestLabels } from "./test-kinds.js";
@@ -41,8 +41,8 @@ export async function renderSpecDiffConformance(options) {
     // state, not to what an interrupted run left half-written. The tests and the whole suite are
     // one unit, landed whole or undone whole.
     const unit = startWork([buildDir, conformanceDir], moduleName);
-    // The notes as its agents find them: a spec gap they add is told with the result.
-    const notesBefore = { code: readNotes(buildDir), tests: readNotes(conformanceDir) };
+    // The spec gaps its agents report from here on are told with the result.
+    const gapsFrom = gapMark();
     const conformance = await renderConformanceTests({
         plainFile: tip.plainFile,
         frid: lastFrid,
@@ -58,8 +58,8 @@ export async function renderSpecDiffConformance(options) {
     });
     // What the codebase now has conformance tests for, as their labels say.
     const tested = () => frids.filter((frid) => missingTestLabels(conformanceDir, frid, []).length === 0);
-    // Read before a rollback takes the notes back: a gap is the person's to decide, whether the tests passed or not.
-    const base = { moduleName, buildDir, conformanceDir, diff: applied, conformance, specGaps: [...linesAddedSince(notesBefore.code, buildDir, "gaps"), ...linesAddedSince(notesBefore.tests, conformanceDir, "gaps")] };
+    // A gap is the person's to decide, whether the tests passed or not.
+    const base = { moduleName, buildDir, conformanceDir, diff: applied, conformance, specGaps: gapsSince(gapsFrom, [buildDir, conformanceDir]) };
     if (!conformance.passed) {
         abandonWork(unit, moduleName, notLandedWhy("for the spec change", conformance.unlabelled));
         return { ...base, tested: tested(), passed: false, stoppedAt: "conformance" };

package/dist/regenerate/conformance-agents.js

@@ -7,6 +7,7 @@ import { confinedTools } from "./confine.js";
 import { ensureInModulesScratch, modulesScratchDir } from "./paths.js";
 import { workspaceBlock } from "./workspace.js";
 import { reportActivity } from "./activity.js";
+import { runTiming, withTestingScripts } from "./test-scripts.js";
 import { createWebTools, WEB_TOOLS } from "./web.js";
 import { diffFiles, digestFiles } from "./codebase.js";
 import { ERROR_CAUSE_RULES, fixMethod, INTEGRITY_RULES, specPrimer } from "./prompt.js";
@@ -377,9 +378,17 @@ export function explorePrompt(buildDir, spec, scope, testingScripts, question, p
         ...(testingScripts !== undefined && testingScripts.length > 0
             ? [
                 "",
-                "This is the project's conformance testing script, which the harness runs to decide whether the",
-                "tests pass - read it for what it builds or installs before the tests run, and so what the tests can depend on:",
-                ...testingScripts.flatMap((script) => ["", `<script path="${script.path}" run-as="${script.runs}">`, script.content.trimEnd(), "</script>"]),
+                ...(testingScripts.some((script) => script.prepare === true)
+                    ? [
+                        "These are the project's prepare script, which the harness runs on the build first, and its conformance testing",
+                        "script, which runs only once the prepare script passes and decides whether the tests pass - read both for what",
+                        "they build or install before the tests run, and so what the tests can depend on:",
+                    ]
+                    : [
+                        "This is the project's conformance testing script, which the harness runs to decide whether the",
+                        "tests pass - read it for what it builds or installs before the tests run, and so what the tests can depend on:",
+                    ]),
+                ...testingScripts.flatMap((script) => ["", `<script${script.prepare === true ? ' kind="prepare environment"' : ""} path="${script.path}" run-as="${script.runs}">`, script.content.trimEnd(), "</script>"]),
             ]
             : []),
         ...(previous !== undefined && question === undefined
@@ -703,7 +712,7 @@ export function unfoundedEvidence(evidence, opened) {
             "opened, the testing script, or the failing run - each with where. What <how to invoke it> says is not evidence: it " +
             "may be wrong, so check it against the testing script and the files.");
     }
-    const inPrompt = (source) => /^(the )?(conformance )?testing script$|^(the )?failing run$/i.test(source.trim());
+    const inPrompt = (source) => /^(the )?(conformance )?testing script$|^(the )?prepare script$|^(the )?failing run$/i.test(source.trim());
     const unread = given.filter((item) => !inPrompt(item.source) && !opened(item.source.trim()));
     if (unread.length > 0) {
         return (`Your evidence cites ${unread.map((item) => item.source.trim()).join(", ")}, which you did not open. Open each file you cite ` +
@@ -734,7 +743,7 @@ export async function judgeFailure(options) {
                 description: "For code: the conformance testing script's own words for what it needs the build to have - the file, entry point or module the run failed on - copied exactly from the script",
             })),
             evidence: Type.Optional(Type.Array(Type.Object({
-                source: Type.String({ description: "Where: the path of a file you opened, `testing script`, or `failing run`" }),
+                source: Type.String({ description: "Where: the path of a file you opened, `testing script`, `prepare script` where there is one, or `failing run`" }),
                 shows: Type.String({ description: "What it shows, e.g. `the script runs node dist/cli.js` or `tsconfig.json emits src/cli.ts as dist/src/cli.js`" }),
             }), { description: "What you found yourself, each with where - never what <how to invoke it> claims unchecked" })),
             functionalities: Type.Optional(Type.Array(Type.Object({
@@ -882,6 +891,8 @@ export function judgePrompt(options) {
         "  the tests set up themselves - a credential, a variable or a fixture a test sets, overrides or names wrong: that is",
         "  tests; nor for a toolchain or package missing where the tests run: the agents install those. The run stops on it",
         `  once it has been judged so ${ENVIRONMENT_RAISES} times in a row; until then the tests are checked for whether they cause it.`,
+        "  A run the time limit stopped is environment only when nothing hangs and it needs more time than a test run may",
+        "  take - and then it stops at once.",
         "",
         "When the output does not show why the software failed - it answered with an error that names no cause, and",
         "logged nothing about it - the code is what is wrong first: it hides the cause, and nobody can fix what they",
@@ -897,6 +908,12 @@ export function judgePrompt(options) {
                 "what is wrong. What it does with the tests, the tests have to fit; what it needs of the code - a file it runs or",
                 "checks for, a command it builds with - the code has to provide. A verdict that the script is wrong is neither",
                 "code nor tests: say which of the two has to change to get past it.",
+                ...(scripts.some((script) => script.prepare === true)
+                    ? [
+                        "The prepare script below it runs on the build before every run, and the conformance testing script only once it",
+                        "passes: it binds the same way - neither agent can change it, and what it needs of the code the code provides.",
+                    ]
+                    : []),
                 "",
             ]
             : []),
@@ -926,13 +943,41 @@ export function judgePrompt(options) {
         "",
         `<how to invoke it>\n${options.invocation.trim()}\n</how to invoke it>`,
         "",
-        ...scripts.flatMap((script) => [`<conformance testing script path="${script.path}" run-as="${script.runs}">`, script.content.trimEnd(), "</conformance testing script>", ""]),
+        ...scripts.flatMap((script) => {
+            const tag = script.prepare === true ? "prepare script" : "conformance testing script";
+            return [`<${tag} path="${script.path}" run-as="${script.runs}">`, script.content.trimEnd(), `</${tag}>`, ""];
+        }),
+        ...(options.run.timedOut === true ? [...timedOutBlock(options.run), ""] : []),
         `<failing run exit="${options.run.exitCode}" command="${options.run.command}">`,
+        runTiming(options.run),
+        "",
         options.run.output.trimEnd(),
         "</failing run>",
         ...(options.escalation !== undefined ? ["", ...escalationBlock(options.escalation)] : []),
     ].join("\n");
 }
+/**
+ * For the judge, of a run the time limit stopped: what kept it from finishing - something that hangs, the code's or a
+ * test's, or a suite that is only slow, which no agent can give more time.
+ */
+export function timedOutBlock(run) {
+    const limit = Math.round(run.timeoutMs / 1000);
+    return [
+        "<timed out>",
+        `This run did not end on its own: the time limit stopped it at ${limit}s, and it counts as failed. Its output ends where it`,
+        "was stopped - the step running then, or the one after the last it printed, is what did not finish. Decide what kept it:",
+        "- code: the software hangs - it waits on input, keeps a server in the foreground, never exits, loops, or blocks on a",
+        "  lock or a call that never answers - when the tests reach it as the specification says.",
+        "- tests: a test waits for something that never comes - a process it started and never stops, output it waits for that",
+        "  the software never prints, a port or a file nobody makes, a sleep or a retry with no end.",
+        "- environment: nothing hangs - the output shows the run making steady progress right up to the limit (installing,",
+        "  building, test after test passing) and it needs more time than a test run may take. No agent can give it that: the",
+        "  run stops at once, and the person is told to raise the limit. Say from the output how far it got, and what took",
+        "  the time.",
+        "Judge from the output and the files, never from the time alone.",
+        "</timed out>",
+    ];
+}
 /** For the judge: the tests were fixed for this failure, and it did not go away - is it the code's? */
 function escalationBlock(escalation) {
     const changed = escalation.changed.length === 0 ? "changed no file" : `changed ${escalation.changed.slice(0, 8).join(", ")}${escalation.changed.length > 8 ? ` and ${escalation.changed.length - 8} more` : ""}`;
@@ -976,7 +1021,10 @@ export async function createCodeFixer(options) {
         model: options.model,
         label: `${options.label} coding agent`, role: "coding",
         system: `${fence.note}\n\n${workspaceBlock({ dir: options.buildDir, hasMyDiff: options.toolNames.includes("my_diff") })}`,
-        task: codeFixerTask(options),
+        // What the conformance testing script needs of the build, the code provides: the fixer reads it, not only a judge's quote of it.
+        task: options.testingScripts !== undefined
+            ? withTestingScripts(codeFixerTask(options), options.testingScripts.scripts, ["conformance", "unit"], options.testingScripts.root)
+            : codeFixerTask(options),
         compaction: { instructions: CODER_COMPACTION },
         ...(options.checkEnvironment !== undefined ? { checkEnvironment: options.checkEnvironment } : {}),
     });
@@ -1029,6 +1077,8 @@ export function codeFixPrompt(options) {
         `<why the code>\n${options.reason.trim()}\n</why the code>`,
         "",
         `<failing run command="${options.run.command}">`,
+        runTiming(options.run),
+        "",
         options.run.output.trimEnd(),
         "</failing run>",
         "",
@@ -1062,6 +1112,8 @@ export function escalatedFixPrompt(options) {
         "</evidence>",
         "",
         `<failing run command="${options.run.command}">`,
+        runTiming(options.run),
+        "",
         options.run.output.trimEnd(),
         "</failing run>",
         "",
@@ -1099,6 +1151,8 @@ export function testsFixPrompt(options) {
         `<why the tests>\n${options.reason.trim()}\n</why the tests>`,
         "",
         "<failing run>",
+        runTiming(options.run),
+        "",
         options.run.output.trimEnd(),
         "</failing run>",
         "",

package/dist/regenerate/conformance.js

@@ -9,12 +9,12 @@ import { abandonWork, diffSinceSnapshot, moduleDirOf, recordHistory, startWork,
 import { createWorkLedger, describeAccounts, SUBMIT_WORK_NOTE, unanswered, unansweredPrompt } from "./submit-work.js";
 import { CONFORMANCE_COMPACTION } from "./compaction.js";
 import { NOTES_TURN, notesForSession, passedOutcome, readNotes, reflectOnReview, withNotes } from "./notes.js";
-import { labelsPrompt, labelsPromptFor, misplacedConformanceTests, misplacedTestsPrompt, missingTestLabels } from "./test-kinds.js";
+import { acceptanceLabel, acceptanceText, labelsPrompt, labelsPromptFor, misplacedConformanceTests, misplacedTestsPrompt, missingTestLabels } from "./test-kinds.js";
 import { createScriptTool, findTestScripts, projectRootFor, requireTestScripts, runConformanceScript, runUnitScript, failureSignature, STUCK_ROUNDS, scriptFailure, staticChecksOnly, withTestingScripts, } from "./test-scripts.js";
 import { createWebTools, WEB_TOOLS } from "./web.js";
 import { createMyDiffTool } from "./self-check.js";
 import { createServiceTools } from "./services.js";
-import { missingToolchainError, toolchainError } from "./environment-error.js";
+import { missingToolchainError, timeLimitError, toolchainError } from "./environment-error.js";
 import { workspaceBlock } from "./workspace.js";
 import { codeFixPrompt, conflictKey, escalatedFixPrompt, createCodeFixer, answerAsExplorer, createAskExplorerTool, exploreInvocation, judgeFailure, SPEC_CONFLICT_RAISES, ENVIRONMENT_RAISES, environmentCheckPrompt, createTestReviewer, testReviewRequest, reviewFeedbackPrompt, testsFixPrompt, sortedRefs, } from "./conformance-agents.js";
 import { resolveRequiresChain } from "./requires.js";
@@ -157,9 +157,11 @@ export async function renderConformanceTests(options) {
     // may read the code but writes only tests. An explorer reads the code first and writes down how the
     // software is invoked, which is what the tests are written against.
     const label = `${moduleName} ${options.diff !== undefined ? "spec change" : frid}`;
-    const specContext = { moduleName, text: specText(specifications, moduleName, frids, required, options.change ?? options.diff, sectionModules(tree, moduleName)) };
     // For a spec change: the functionalities it touches, whose acceptance tests are labelled, checked and reviewed.
     const touched = forChange ? touchedFunctionalities(tree, frids, options.diff) : [];
+    // The acceptance tests the run's tests carry out: every functionality the change touches, or the one under test.
+    const carriedOut = forChange ? touched : [{ frid: frid, acceptanceTests: specifications[ACCEPTANCE_TESTS] ?? [] }];
+    const specContext = { moduleName, text: specText(specifications, moduleName, frids, required, carriedOut, options.change ?? options.diff, sectionModules(tree, moduleName)) };
     // Any functionality of the codebase - this module's, and those of the modules it requires. A
     // functionality's run may raise a spec conflict only with that functionality; a spec diff's, between any two.
     const own = (forChange ? frids : frids.slice(0, frids.indexOf(frid) + 1)).map((id) => ({ module: moduleName, frid: id }));
@@ -223,11 +225,15 @@ export async function renderConformanceTests(options) {
     };
     stage(`exploring the code for how it is invoked${options.model ? ` (${options.model})` : ""}`);
     // The explorer reads the conformance testing script too: it is how the tests are built and run, and so how they reach the software.
+    // The prepare script, where there is one, runs on the build before it - and only once it passes does the suite run.
     const testingScripts = [
+        ...(scripts.prepare !== undefined
+            ? [{ path: relative(root, scripts.prepare), runs: "<script> <copy of the build>, before every run of the conformance testing script", content: readFileSync(scripts.prepare, "utf8"), prepare: true }]
+            : []),
         { path: relative(root, scripts.conformance), runs: "<script> <copy of the build> <copy of the whole conformance suite>", content: readFileSync(scripts.conformance, "utf8") },
     ];
     // The acceptance tests the explorer's plan has to carry out, step by step.
-    const toCarryOut = (forChange ? touched : [{ frid: frid, acceptanceTests: specifications[ACCEPTANCE_TESTS] ?? [] }])
+    const toCarryOut = carriedOut
         .map((item) => ({ module: moduleName, frid: item.frid, count: item.acceptanceTests.length }))
         .filter((item) => item.count > 0);
     let invocation = await exploreInvocation({ buildDir, spec: specContext, scope, testingScripts, acceptance: toCarryOut, label, model: options.model, onDelta: options.onDelta });
@@ -272,6 +278,7 @@ export async function renderConformanceTests(options) {
         cwd: conformanceDir,
         customTools: [
             notes.tool,
+            notes.gapTool,
             work.tool,
             ...createWebTools(),
             ...fence.tools,
@@ -285,7 +292,7 @@ export async function renderConformanceTests(options) {
             }),
             createCopyResourceTool({ resources: linkedResources, dir: conformanceDir }),
         ],
-        tools: [...fence.names, "run_conformance_tests", "my_diff", "ask_explorer", "update_notes", "submit_work", "copy_resource", ...WEB_TOOLS],
+        tools: [...fence.names, "run_conformance_tests", "my_diff", "ask_explorer", "update_notes", "report_spec_gap", "submit_work", "copy_resource", ...WEB_TOOLS],
         model: options.model,
         label: `${label} conformance agent`,
         role: "conformance",
@@ -313,12 +320,14 @@ export async function renderConformanceTests(options) {
             // The unit tests, and the conformance suite - to reproduce the failure and check the fix, seeing only its output.
             tools: [
                 fixerNotes.tool,
+                fixerNotes.gapTool,
                 createScriptTool({ kind: "unit", script: scripts.unit, run: () => runUnitScript(buildDir, scripts.unit), runs: unitRuns }),
                 fixerConformanceRuns.tool,
                 ...fixerServices.tools,
             ],
-            toolNames: ["run_unit_tests", "run_conformance_tests", ...fixerServices.names, "update_notes"],
+            toolNames: ["run_unit_tests", "run_conformance_tests", ...fixerServices.names, "update_notes", "report_spec_gap"],
             checkEnvironment: { run: () => runUnitScript(buildDir, scripts.unit), script: scripts.unit },
+            testingScripts: { scripts, root },
         });
         fixerConformanceRuns.newTurn();
         return ask(fixer, withNotes(text, buildDir), options.onDelta);
@@ -494,19 +503,6 @@ export async function renderConformanceTests(options) {
                         idleFixes = 0;
                     continue;
                 }
-                // A run out of time is not a question of test or code: the tests' environment is the conformance agent's.
-                if (recheck.timedOut === true) {
-                    const moved = await unmoved();
-                    if (moved === "stuck")
-                        break;
-                    if (moved === "asked")
-                        continue;
-                    stage("the conformance run did not get to the tests - back to the conformance agent");
-                    const fixed = await changing(() => askConformance(scriptFailure(recheck, "conformance testing"), options.onDelta));
-                    response = fixed.result;
-                    recordAttempt(rounds, recheck, `not a question of code or tests (${recheck.timedOut === true ? "the time limit" : `exit ${recheck.exitCode}`})`, { who: "conformance agent", files: fixed.files });
-                    continue;
-                }
                 stage(`conformance tests failing - working out whether the code, the tests or the spec are wrong${judgeModel ? ` (${judgeModel})` : ""}`);
                 // A fix that changed no file leaves the next run as this one: IDLE_FIXES of them in a row, and nothing will change.
                 const idle = (files) => {
@@ -541,6 +537,14 @@ export async function renderConformanceTests(options) {
                     onDelta: options.onDelta,
                 });
                 const reason = verdict.reason.split("\n")[0].slice(0, 160);
+                // Stopped by the time limit while it was still getting on: no agent can give it more time. The pulse stops now -
+                // rolled back, no retry - and the person is told how to raise the limit.
+                if (verdict.culprit === "environment" && recheck.timedOut === true) {
+                    stage(`the conformance run needs more than the ${Math.round(recheck.timeoutMs / 1000)}s a test run may take - stopping`);
+                    if (start !== undefined)
+                        abandonWork(start, moduleName, "the conformance testing script needs more time than a test run may take");
+                    throw timeLimitError(recheck, { moduleName, script: scripts.conformance, kind: "conformance", reason: verdict.reason });
+                }
                 // The environment - what the run is given - is the person's to fix: once judged so twice in a row, the run stops.
                 if (verdict.culprit === "environment") {
                     environmentRaises += 1;
@@ -692,7 +696,7 @@ export async function renderConformanceTests(options) {
                     buildDir,
                     conformanceDir,
                     spec: specContext,
-                    functionalities: forChange ? touched : [{ frid: frid, acceptanceTests: specifications[ACCEPTANCE_TESTS] ?? [] }],
+                    functionalities: carriedOut,
                     forChange,
                     invocation,
                     label,
@@ -952,9 +956,10 @@ export function failureLine(run) {
 }
 /**
  * The spec as the explorer, the judge and the coding agent are shown it - its functional specs
- * numbered with their ids, so a spec conflict can name them.
+ * numbered with their ids, so a spec conflict can name them, and the acceptance tests the run carries
+ * out labelled as their tests are, so a verdict can quote them.
  */
-function specText(specifications, moduleName, frids, required, diff, modules) {
+function specText(specifications, moduleName, frids, required, acceptance, diff, modules) {
     // Every functionality in the codebase, each with its module: the required modules' first, then this one's.
     const numbered = (key, items) => key === FUNCTIONAL_REQUIREMENTS
         ? [
@@ -962,10 +967,14 @@ function specText(specifications, moduleName, frids, required, diff, modules) {
         ]
         : items;
     // Each section in its tag, as every agent is shown it (see specPrimer).
-    const sections = [DEFINITIONS, NON_FUNCTIONAL_REQUIREMENTS, TEST_REQUIREMENTS, FUNCTIONAL_REQUIREMENTS, ACCEPTANCE_TESTS]
+    const sections = [DEFINITIONS, NON_FUNCTIONAL_REQUIREMENTS, TEST_REQUIREMENTS, FUNCTIONAL_REQUIREMENTS]
         .map((key) => [key, specifications[key] ?? []])
         .filter(([, items]) => items.length > 0)
-        .map(([key, items]) => (key === FUNCTIONAL_REQUIREMENTS || key === ACCEPTANCE_TESTS ? section(key, numbered(key, items)) : moduleSection(key, items, moduleName, modules)));
+        .map(([key, items]) => (key === FUNCTIONAL_REQUIREMENTS ? section(key, numbered(key, items)) : moduleSection(key, items, moduleName, modules)));
+    // Not only the last functionality's: a spec change's run carries out the acceptance tests of every one it touches.
+    const tests = acceptance.flatMap((item) => item.acceptanceTests.map((test, index) => `${acceptanceLabel(item.frid, index + 1)}: ${acceptanceText(test)}`));
+    if (tests.length > 0)
+        sections.push(section(ACCEPTANCE_TESTS, tests));
     if (diff !== undefined)
         sections.push(specChangeBlock(diff));
     return sections.join("\n\n");

package/dist/regenerate/debugger.js

@@ -4,6 +4,7 @@ import { Type } from "typebox";
 import { ask, BEGIN, createExecutor } from "../pi.js";
 import { confinedTools } from "./confine.js";
 import { createWebTools, WEB_TOOLS } from "./web.js";
+import { runTiming } from "./test-scripts.js";
 /**
  * A fresh look at a failure the coding agent keeps hitting.
  *
@@ -24,9 +25,12 @@ export function diagnosisText(diagnosis) {
 }
 function debugPrompt(input) {
     const script = input.script !== undefined && existsSync(input.script) ? readFileSync(input.script, "utf8") : undefined;
+    const timedOut = input.run?.timedOut === true;
     return [
-        `The unit tests of the codebase in ${input.buildDir} have failed the same way ${input.times} times in a row. The coding agent`,
-        "keeps changing things and the failure does not move: it has not seen the cause. You look at it with fresh eyes.",
+        timedOut
+            ? `The unit testing script of the codebase in ${input.buildDir} did not finish: the time limit stopped it. You look at it with fresh eyes.`
+            : `The unit tests of the codebase in ${input.buildDir} have failed the same way ${input.times} times in a row. The coding agent`,
+        ...(timedOut ? [] : ["keeps changing things and the failure does not move: it has not seen the cause. You look at it with fresh eyes."]),
         "",
         "Find the root cause - the one thing that makes this failure happen - and the change that fixes it. Read the failing",
         "run below, the tests it names, the code they exercise, the build files (pyproject.toml, package.json, go.mod, pom.xml,",
@@ -37,6 +41,21 @@ function debugPrompt(input) {
         "",
         "You change nothing - you only read. When you know, call submit_diagnosis: the root cause, with the evidence for it,",
         "and the fix - which files, and exactly what to change in them.",
+        ...(timedOut
+            ? [
+                "",
+                "<timed out>",
+                "The run's output ends where the time limit stopped it: the test running then, or the step after the last one it",
+                "printed, is what did not finish. Find what kept it:",
+                "- something hangs - a test or the code waits on input, keeps a server in the foreground, never exits, loops, blocks",
+                "  on a lock or a network call that never answers: the root cause and the fix, as for any failure;",
+                "- nothing hangs - the output shows the run making steady progress right up to the limit (installing, building, test",
+                "  after test passing) and it needs more time than a test run may take: set too_slow, and say in root_cause from the",
+                "  output how far it got and what took the time. No agent can give it more time: the work stops, and the person is",
+                "  told to raise the limit. Judge from the output and the files, never from the time alone.",
+                "</timed out>",
+            ]
+            : []),
         "",
         "<what the change is meant to do>",
         input.task.trim(),
@@ -44,6 +63,7 @@ function debugPrompt(input) {
         ...(script !== undefined ? ["", `<testing script path="${input.script}">`, script.trimEnd(), "</testing script>"] : []),
         "",
         "<failing run>",
+        ...(input.run !== undefined ? [runTiming(input.run), ""] : []),
         input.failure.trimEnd().slice(-8000),
         "</failing run>",
     ].join("\n");
@@ -58,10 +78,14 @@ export async function debugFailure(input) {
         parameters: Type.Object({
             root_cause: Type.String({ description: "The one thing that makes the failure happen, and the evidence for it: the lines of the run, the code, the docs" }),
             fix: Type.String({ description: "The change that fixes it: which files, and exactly what to change in them" }),
+            too_slow: Type.Optional(Type.Boolean({
+                description: "Only for a run the time limit stopped: true when nothing hangs - it was making progress up to the limit and needs more time than a test run may take",
+            })),
         }),
         async execute(_id, params) {
             const given = params;
-            found = { rootCause: given.root_cause, fix: given.fix };
+            // Too slow is a verdict on a run the limit stopped, and on no other.
+            found = { rootCause: given.root_cause, fix: given.fix, ...(given.too_slow === true && input.run?.timedOut === true ? { tooSlow: true } : {}) };
             return { content: [{ type: "text", text: "Recorded." }], details: {}, terminate: true };
         },
     });