@turri-dev/cli 0.1.4-next.318a98a → 0.1.4-next.5a94e68

package/dist/regenerate/activity.js

@@ -90,6 +90,27 @@ export function onModelEvent(listener) {
     modelListeners.add(listener);
     return () => modelListeners.delete(listener);
 }
+/**
+ * The model provider failing or stalling, in a line for the person: the chat's notification, a headless run's console.
+ * A turn turri stopped itself is the work going its normal course - kept in the activity, the pulse's log and the run
+ * report, not said.
+ */
+export function describeTurnIncident(event) {
+    const who = event.role ?? event.label ?? "an agent";
+    const model = event.model !== undefined ? ` (${event.model})` : "";
+    if (event.kind === "provider_retry") {
+        const wait = Math.round((event.waitMs ?? 0) / 1000);
+        return `The model provider failed${model}: ${(event.error ?? "").slice(0, 160)} - trying ${who}'s request again in ${wait}s (${event.attempt} of ${event.of}).`;
+    }
+    if (event.kind === "provider_stall") {
+        return `The model provider has sent nothing back to ${who}${model} for ${formatSilence(event.waitMs ?? 0)} - still waiting.`;
+    }
+    return undefined;
+}
+function formatSilence(ms) {
+    const seconds = Math.round(ms / 1000);
+    return seconds >= 60 && seconds % 60 === 0 ? `${seconds / 60}m` : seconds >= 60 ? `${Math.floor(seconds / 60)}m ${seconds % 60}s` : `${seconds}s`;
+}
 export function reportModelEvent(event) {
     if (modelListeners.size === 0)
         return;

package/dist/regenerate/list-arguments.js

@@ -0,0 +1,121 @@
+const isObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+/** The tool's arguments that take a list of objects. */
+function listArguments(schema) {
+    const required = new Set(schema.required ?? []);
+    return Object.entries(schema.properties ?? {})
+        .filter(([, property]) => property.type === "array" && property.items?.type === "object")
+        .map(([name, property]) => ({ name, required: required.has(name), item: property.items }));
+}
+/** JSON text, fenced or not, as what it holds; anything else as it is. */
+function parsed(value) {
+    if (typeof value !== "string")
+        return value;
+    const text = value
+        .trim()
+        .replace(/^```(?:json)?\s*/i, "")
+        .replace(/\s*```$/, "");
+    if (!/^[[{]/.test(text))
+        return value;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return value;
+    }
+}
+const escaped = (name) => name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+/**
+ * Arguments the model wrote inside another argument's text - `answer</a>\n<b>[...]</b>\n<c>...` - taken out of it and
+ * given as their own, where they were not given already. Which argument each was found in, for the refusal.
+ */
+function untucked(args, names) {
+    const out = { ...args };
+    const foundIn = new Map();
+    const anyTag = names.map(escaped).join("|");
+    for (const [key, value] of Object.entries(args)) {
+        if (typeof value !== "string")
+            continue;
+        let cut = value.search(new RegExp(`</${escaped(key)}>`));
+        for (const name of names) {
+            if (name === key || (out[name] !== undefined && out[name] !== ""))
+                continue;
+            const match = new RegExp(`<${escaped(name)}>([\\s\\S]*?)(?:</${escaped(name)}>|(?=<(?:${anyTag})>)|$)`).exec(value);
+            if (match === null)
+                continue;
+            out[name] = match[1].trim();
+            foundIn.set(name, key);
+            cut = cut === -1 ? match.index : Math.min(cut, match.index);
+        }
+        if (cut !== -1)
+            out[key] = value.slice(0, cut).trim();
+    }
+    return { args: out, foundIn };
+}
+/** What an entry of the list looks like, as JSON: its fields, the optional ones marked. */
+function shapeOf(item) {
+    const required = new Set(item.required ?? []);
+    const entries = Object.entries(item.properties ?? {});
+    const fields = entries.map(([name, field]) => `"${name}"${required.has(name) ? "" : "?"}: ${field.type ?? "value"}`).join(", ");
+    const sample = (field) => (field.type === "number" || field.type === "integer" ? "1" : field.type === "boolean" ? "true" : field.type === "string" ? '"..."' : "...");
+    const example = entries
+        .filter(([name]) => required.has(name))
+        .map(([name, field]) => `"${name}": ${sample(field)}`)
+        .join(", ");
+    return { fields: `{${fields}}`, example: `{${example}}` };
+}
+/** Why a list argument is not one, said so a model can make the call right next time - or undefined when it is. */
+function problemOf(list, value, foundIn) {
+    if (value === undefined)
+        return list.required ? "you left it out" : undefined;
+    if (!Array.isArray(value)) {
+        if (foundIn !== undefined)
+            return `you wrote it inside \`${foundIn}\``;
+        return typeof value === "string" ? "you sent it as text, not a list" : `you sent ${isObject(value) ? "an object" : `a ${typeof value}`}, not a list`;
+    }
+    const bad = value.findIndex((entry) => !isObject(entry));
+    if (bad === -1)
+        return undefined;
+    return `entry ${bad + 1} is ${typeof value[bad] === "string" ? "text" : `a ${typeof value[bad]}`}, not an object`;
+}
+/** The arguments with their lists made lists where they plainly are; refused, saying what is wanted, where they are not. */
+export function prepareListArguments(toolName, schema, raw) {
+    const lists = listArguments(schema);
+    if (lists.length === 0 || !isObject(raw))
+        return raw;
+    const { args, foundIn } = untucked(raw, Object.keys(schema.properties ?? {}));
+    const problems = [];
+    for (const list of lists) {
+        let value = parsed(args[list.name]);
+        if (isObject(value))
+            value = [value];
+        if (Array.isArray(value))
+            value = value.map(parsed);
+        if (value !== undefined)
+            args[list.name] = value;
+        const problem = problemOf(list, value, foundIn.get(list.name));
+        if (problem === undefined)
+            continue;
+        const { fields, example } = shapeOf(list.item);
+        problems.push(`- \`${list.name}\`: ${problem}. It is a list of objects, each ${fields} - e.g. "${list.name}": [${example}].`);
+    }
+    if (problems.length === 0)
+        return args;
+    throw new Error([
+        `${toolName} was not run - its arguments are not in the shape it takes:`,
+        ...problems,
+        "Each argument is a field of its own in the call, at the top level: a list is a JSON array of objects, never text, and never written inside another argument's text.",
+    ].join("\n"));
+}
+/** The tools, each that takes a list of objects preparing its arguments so before they are checked. */
+export function withListArguments(tools) {
+    return tools.map((tool) => {
+        const schema = tool.parameters;
+        if (listArguments(schema).length === 0)
+            return tool;
+        const before = tool.prepareArguments;
+        return {
+            ...tool,
+            prepareArguments: (args) => prepareListArguments(tool.name, schema, before !== undefined ? before(args) : args),
+        };
+    });
+}

package/dist/regenerate/loop.js

@@ -597,7 +597,14 @@ export function createWorkLoop(options) {
                 }
                 return {
                     report: released !== undefined ? `${report}\n\n${released}` : report,
-                    details: { passed: result.landed, landed: result.landed, ...(released !== undefined ? { released: true } : {}), ...(result.story !== undefined ? { story: result.story } : {}) },
+                    details: {
+                        passed: result.landed,
+                        landed: result.landed,
+                        ...(released !== undefined ? { released: true } : {}),
+                        ...(result.story !== undefined ? { story: result.story } : {}),
+                        // The spec gaps its agents met, for a headless run to say - in the chat the report puts them to the person.
+                        ...(result.specGaps !== undefined && result.specGaps.length > 0 ? { specGaps: result.specGaps } : {}),
+                    },
                 };
             },
         });
@@ -1240,17 +1247,21 @@ export function createWorkLoop(options) {
                 startAgain(entry, `it now requires ${requires.join(", ")}, which its code was not built on`);
             return loop.bringUp(plainFile, request);
         },
-        async fromScratch(plainFile, request = {}) {
+        async fromScratch(plainFile, { chain, ...request } = {}) {
             const file = resolve(options.cwd, plainFile);
-            // Where its code lives: the tips its module is built in, or - none built yet - its own codebase.
-            const codebases = chainsFor(file).map(tipOf);
+            // Where its code lives: the tips its module is built in, or - none built yet - its own codebase. The whole chain:
+            // every codebase in those chains, bottom first - a tip taken back to nothing is otherwise started from a copy of
+            // the code below it.
+            const chains = chainsFor(file);
+            const codebases = [...new Map((chain === true ? chains.flat() : chains.map(tipOf)).map((entry) => [entry.buildDir, entry])).values()];
             for (const entry of codebases) {
                 if (modules.get(entry.buildDir)?.running !== undefined)
                     throw new Error(`${entry.moduleName} has a pulse running: halt it, or ask again once it is done.`);
             }
+            const why = chain === true ? "asked to be regenerated from scratch, with its whole chain" : "asked to be regenerated from scratch";
             for (const entry of codebases)
                 if (started(entry))
-                    startAgain(entry, "asked to be regenerated from scratch");
+                    startAgain(entry, why);
             return loop.bringUp(plainFile, request);
         },
         halt(target, reason) {
@@ -1511,7 +1522,7 @@ export function describeModules(statuses, options = {}) {
                 `  Ask the user whether to start ${codebases.join(", ")} over, and say why: ${codebases.join(", ")} was built without ` +
                     `${requires.join(", ")}, so no change to its code can bring ${requires.join(", ")} in - its code and conformance ` +
                     "tests are taken out, kept in its history, and built again from the bottom of the chain. Only if they agree, " +
-                    `call regenerate with ${basename(status.plainFile)} and from_scratch.`,
+                    `call regenerate with ${basename(status.plainFile)} and from_scratch: "module".`,
             ];
     };
     return statuses
@@ -1565,7 +1576,7 @@ const defaultConformanceRunner = async (job, onDelta) => {
         writeTestsRecord(result.conformanceDir, job.planned.step.tree);
         recordHistory(job.buildDir, `conformance tests: ${job.planned.title}`);
     }
-    return { landed: result.passed, report: describeSpecDiffConformance(result) };
+    return { landed: result.passed, report: describeSpecDiffConformance(result), ...(result.specGaps !== undefined ? { specGaps: result.specGaps } : {}) };
 };
 /** A job as a spec diff render, landed on its unit tests: conformance is a track of its own. */
 const defaultRunner = async (job, onDelta) => {
@@ -1581,7 +1592,9 @@ const defaultRunner = async (job, onDelta) => {
         onDelta,
     });
     // Landed code can be tried.
-    return { landed: result.passed, report: withTryItLine(describeSpecDiff(result), result.passed && result.restructured !== true), story: codeStory(result) };
+    return { landed: result.passed, report: withTryItLine(describeSpecDiff(result), result.passed && result.restructured !== true), story: codeStory(result),
+        ...(result.specGaps !== undefined ? { specGaps: result.specGaps } : {}),
+    };
 };
 /** How a code pulse went, for its line in the notes: the rounds it took, its test runs, what the debugger found. */
 export function codeStory(result) {

package/dist/regenerate/regenerate.js

@@ -281,9 +281,11 @@ export async function renderSpecDiff(options) {
     const work = createWorkLedger();
     // What it checks its own work with: the diff its reviewer is shown, and whether a test catches a bug - neither decides anything.
     const gate = createUnitGate(buildDir, scripts.unit);
+    // can_it_fail, for the coding agent and its reviewer: on unless mutation-testing says otherwise.
+    const mutationTesting = scripts.unit !== undefined && configForSpec(tip.plainFile).mutationTesting !== false;
     const selfChecks = [
         createMyDiffTool({ dir: buildDir, what: "code" }),
-        ...(scripts.runOn === "copy" ? [createCanItFailTool({ buildDir, script: scripts.unit })] : []),
+        ...(mutationTesting ? [createCanItFailTool({ buildDir, script: scripts.unit })] : []),
     ];
     // A server it built, it can start and try from outside - stopped when it is done.
     const services = createServiceTools({ dir: buildDir });
@@ -499,7 +501,7 @@ export async function renderSpecDiff(options) {
                 forChange: true,
                 ...(options.cases !== undefined ? { cases: options.cases } : {}),
                 label: `${moduleName} spec change`,
-                ...(scripts.runOn === "copy" ? { unitScript: scripts.unit } : {}),
+                ...(mutationTesting ? { unitScript: scripts.unit } : {}),
                 testingScript: { path: relative(root, scripts.unit), content: readFileSync(scripts.unit, "utf8") },
                 model: options.judgeModel ?? modelSpec(inheritModel(configForSpec(tip.plainFile).judgeModel, modelOnly(options.model))),
                 send: async (next) => {

package/dist/regenerate/self-check.js

@@ -1,9 +1,9 @@
-import { readFileSync, writeFileSync } from "node:fs";
-import { join, relative, resolve, sep } from "node:path";
+import { readFileSync } from "node:fs";
+import { relative, resolve, sep } from "node:path";
 import { defineTool } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
 import { workDiff } from "./lineage.js";
-import { lastPassingMs, runOnCode } from "./test-scripts.js";
+import { lastPassingMs, runOnCode, scriptsRunInPlace } from "./test-scripts.js";
 /**
  * Tools an agent checks its own work with, before the harness and the reviewer do: the diff the reviewer is shown, and
  * whether a test catches a change to the code. Neither decides anything - the testing scripts, run by the harness, still do.
@@ -43,16 +43,23 @@ export function canItFailLimitMs(buildDir, script) {
  */
 export function createCanItFailTool(options) {
     let left = options.runs ?? CAN_IT_FAIL_RUNS;
+    // In place, the change is made to the code itself for the run, and undone after it: the scripts run on the code there.
+    const inPlace = scriptsRunInPlace(options.buildDir);
     return defineTool({
         name: "can_it_fail",
         label: "Can it fail",
         description: "Check that the unit tests catch a bug: give one change to the code that breaks a behaviour the specification " +
             "states - a value or a status changed, a check taken out, a condition turned round where it cannot make a loop run for " +
             "ever - and the unit testing script " +
-            "runs on a copy of the code with that change. The code itself is never touched. The tests should fail; if they all " +
+            (inPlace
+                ? "runs with that change made to the code, which is put back as it was once the run ends, whatever its result. "
+                : "runs on a copy of the code with that change. The code itself is never touched. ") +
+            "The tests should fail; if they all " +
             "pass, no test catches that bug. Make a change that still builds and runs: a failure only because it no longer " +
             `compiles or imports says nothing about the tests. ${options.runs ?? CAN_IT_FAIL_RUNS} runs at most.`,
-        promptSnippet: "can_it_fail: whether the unit tests catch a bug you put in a copy of the code",
+        promptSnippet: "can_it_fail: whether the unit tests catch a bug you put in the code for one run",
+        // Never beside another call: in place, an edit made while the bug is in would be undone with it.
+        executionMode: "sequential",
         parameters: Type.Object({
             file: Type.String({ description: "The code file to change, relative to the code folder" }),
             find: Type.String({ description: "Text that is in the file exactly once - the code to change" }),
@@ -83,7 +90,7 @@ export function createCanItFailTool(options) {
             left -= 1;
             let run;
             try {
-                run = await runOnCode(options.buildDir, options.script, "unit testing script, with the change", (copy) => writeFileSync(join(copy, relative(options.buildDir, path)), text.slice(0, at) + replace + text.slice(at + find.length)), canItFailLimitMs(options.buildDir, options.script));
+                run = await runOnCode(options.buildDir, options.script, "unit testing script, with the change", { file: relative(options.buildDir, path), text: text.slice(0, at) + replace + text.slice(at + find.length) }, canItFailLimitMs(options.buildDir, options.script));
             }
             catch (error) {
                 return say(error instanceof Error ? error.message : String(error));

package/dist/regenerate/test-scripts.js

@@ -297,6 +297,10 @@ export function setScriptTimeoutForTests(ms) {
  * own paths start - and whether they run on the code in place (the default), rather than on a fresh copy
  * (`testing-scripts-run-on: copy`).
  */
+/** Whether the testing scripts run on the code in `buildDir` itself, not on a copy of it. */
+export function scriptsRunInPlace(buildDir) {
+    return projectOfBuild(buildDir).inPlace;
+}
 function projectOfBuild(buildDir) {
     const parts = resolve(buildDir).split(sep);
     const index = parts.lastIndexOf("plain_modules");
@@ -401,30 +405,70 @@ const lastPassing = new Map();
 export function lastPassingMs(buildDir, script) {
     return lastPassing.get(`${resolve(buildDir)}\n${script}`);
 }
+/** Files changed in place for a run under way, and what each held: put back however the run ends - the process exiting too. */
+const changedInPlace = new Map();
+let restoreOnExit = false;
+function putBack(path) {
+    const original = changedInPlace.get(path);
+    if (original === undefined)
+        return;
+    writeFileSync(path, original);
+    changedInPlace.delete(path);
+}
+/**
+ * `run` with `change` made to the code itself, and undone once it ends - passed, failed, timed out, thrown, or the process
+ * exiting. Killed outright, the change is left: the next unit of work starts from the last commit (lineage.ts startWork).
+ */
+async function withChangeInPlace(buildDir, change, run) {
+    const path = resolve(buildDir, change.file);
+    if (changedInPlace.has(path))
+        throw new Error(`${change.file} is being changed for another run already.`);
+    if (!restoreOnExit) {
+        restoreOnExit = true;
+        process.once("exit", () => {
+            for (const changed of [...changedInPlace.keys()])
+                putBack(changed);
+        });
+    }
+    changedInPlace.set(path, readFileSync(path));
+    try {
+        writeFileSync(path, change.text);
+        return await run();
+    }
+    finally {
+        putBack(path);
+    }
+}
 /**
- * A script that takes the code folder - the unit testing script - run where the testing scripts
- * run. `change`: what to do to the fresh copy before it runs - only where there is a copy; never in place.
+ * A script that takes the code folder - the unit testing script - run where the testing scripts run. `change`: one file
+ * as it is for this run - made to the fresh copy, or, in place, to the code itself and undone once the run ends.
  */
 export async function runOnCode(buildDir, script, what, change, 
 /** A time limit of its own, in place of a test run's. */
 limitMs) {
     const project = projectOfBuild(buildDir);
+    const passed = (run, started) => {
+        if (change === undefined && run.exitCode === 0)
+            lastPassing.set(`${resolve(buildDir)}\n${script}`, Date.now() - started);
+        return run;
+    };
     if (project.inPlace) {
-        if (change !== undefined)
-            throw new Error("This runs only where the testing scripts run on a copy: here they run in place, on the code itself.");
-        return runScript(project.root, script, [resolve(buildDir)], buildDir);
+        const inPlace = () => runScript(project.root, script, [resolve(buildDir)], buildDir, limitMs !== undefined ? { limitMs } : {});
+        const started = Date.now();
+        if (change === undefined)
+            return passed(await inPlace(), started);
+        reportActivity(`  changing ${change.file} in place for the ${what} - put back once it has run`);
+        return withChangeInPlace(buildDir, change, inPlace);
     }
     const staging = stagingFor(buildDir);
     rmSync(staging.root, { recursive: true, force: true });
     ensureInModulesScratch(buildDir, staging.root);
     try {
         reportActivity(`  copying ${stageBuild(buildDir, staging.build)} files of the code for the ${what}`);
-        change?.(staging.build);
+        if (change !== undefined)
+            writeFileSync(join(staging.build, change.file), change.text);
         const started = Date.now();
-        const run = await runOnCopy(staging.root, script, [staging.build], buildDir, limitMs);
-        if (change === undefined && run.exitCode === 0)
-            lastPassing.set(`${resolve(buildDir)}\n${script}`, Date.now() - started);
-        return run;
+        return passed(await runOnCopy(staging.root, script, [staging.build], buildDir, limitMs), started);
     }
     finally {
         rmSync(staging.root, { recursive: true, force: true });

package/dist/regenerate/unit-review.js

@@ -96,7 +96,7 @@ export function unitReviewTask(options) {
                 "  assertion so loose any answer passes, an expected value computed the way the code computes it.",
                 ...(options.canItFail === true
                     ? [
-                        "- A test you doubt can fail, check: can_it_fail puts a bug in a copy of the code - a condition turned round, a",
+                        "- A test you doubt can fail, check: can_it_fail puts a bug in the code for one run - a condition turned round, a",
                         "  value changed - and runs the unit tests on it. A test that still passes cannot fail: raise it as bad, with the",
                         `  bug it missed. ${REVIEWER_CAN_IT_FAIL_RUNS} runs a session: spend them on the tests the work rests on.`,
                     ]

package/dist/regenerate-cli.js

@@ -19,10 +19,15 @@ its conformance tests too when they are on.
   --start-over             start over, without asking, a codebase regenerated without a module it now requires -
                            with no terminal to ask and without it, the run stops there
   --from-scratch           delete the module's code and conformance tests (its lineage keeps them) and regenerate all of it
+  --from-scratch-chain     the same for every module of its requires chain, built again from the bottom - after
+                           modules were split or moved
   --harness-model <p/id>   config.turri.yaml's harness-model for this run: the planner and the conformance chooser
   --coding-model <p/id>    its coding-model, --model too: the agents that build; p/id:level for how hard a model thinks
   --judge-model <p/id>     its judge-model: the judge and the test review
   --review <true|false>    its review: whether the code and its tests are reviewed once they pass
+  --mutation-testing <true|false>
+                           its mutation-testing: whether the coding agent and the reviewer can put a bug in the code
+                           for one test run (can_it_fail), to see that a test catches it
   --conformance-tests <all|chosen|module|false>
                            its conformance-tests: which conformance tests are written and run
   --build-dir <dir>        as the run ends, however it ends, export the code there: the furthest-built codebase's
@@ -31,7 +36,7 @@ its conformance tests too when they are on.
                            as it starts, as each pulse ends and as it exits (docs/headless.md)
 
   PLAIN_TEMPLATE_DIRS      colon-separated template directories`;
-const { positional, options } = parseArgs(process.argv.slice(2), ["--dry-run", "--start-over", "--from-scratch"], USAGE, ["--build-dir", "--report", "--conformance-tests", "--harness-model", "--judge-model", "--review"]);
+const { positional, options } = parseArgs(process.argv.slice(2), ["--dry-run", "--start-over", "--from-scratch", "--from-scratch-chain"], USAGE, ["--build-dir", "--report", "--conformance-tests", "--harness-model", "--judge-model", "--review", "--mutation-testing"]);
 const { report, buildDir } = headlessOptions(options, true);
 const named = positional[0] ?? readProjectConfig(process.cwd()).module;
 // Functional specs named after the file: regenerated again though their spec did not change.
@@ -40,9 +45,10 @@ if (!named) {
     console.error(`${USAGE}\n\nNo file given, and ${CONFIG_FILE} names no module.`);
     process.exit(1);
 }
+const scratch = options["from-scratch-chain"] !== undefined ? "chain" : options["from-scratch"] !== undefined ? "module" : undefined;
 // A dry run starts nothing, and from scratch deletes before it plans: the two do not go together.
-if (options["from-scratch"] !== undefined && options["dry-run"] !== undefined) {
-    console.error("--from-scratch deletes the module's code before anything is planned: it cannot be a dry run.");
+if (scratch !== undefined && options["dry-run"] !== undefined) {
+    console.error(`--from-scratch${scratch === "chain" ? "-chain" : ""} deletes the code before anything is planned: it cannot be a dry run.`);
     process.exit(1);
 }
 runLoop({
@@ -54,8 +60,8 @@ runLoop({
     startOver: options["start-over"] !== undefined,
     ask: (loop) => again.length > 0
         ? loop.regenerateAgain(named, again)
-        : options["from-scratch"] !== undefined
-            ? loop.fromScratch(named)
+        : scratch !== undefined
+            ? loop.fromScratch(named, { chain: scratch === "chain" })
             : loop.bringUp(named),
 }).catch((error) => {
     console.error(error instanceof Error ? error.message : error);

package/dist/review/live.js

@@ -79,7 +79,8 @@ export function createLiveReviewer(opts) {
             pending.push(finding);
             all.push(finding);
         }
-        if (violations.length === 0) {
+        // No verdict - its turn stopped before it gave one - clears nothing: the scope is looked at again next pass.
+        if (captured !== undefined && violations.length === 0) {
             cleared.push(scope);
         }
         else if (captured?.question) {

package/dist/run-report.js

@@ -22,6 +22,9 @@ const emptyTotals = () => ({
     llm_calls: 0,
     agent_turns: 0,
     compactions: 0,
+    provider_retries: 0,
+    provider_stalls: 0,
+    turns_stopped: 0,
     model_seconds: 0,
 });
 function addUsage(totals, usage) {
@@ -43,8 +46,14 @@ function addEvent(totals, event, tokens) {
     }
     else if (event.kind === "turn")
         totals.agent_turns += 1;
-    else
+    else if (event.kind === "compaction")
         totals.compactions += 1;
+    else if (event.kind === "provider_retry")
+        totals.provider_retries += 1;
+    else if (event.kind === "provider_stall")
+        totals.provider_stalls += 1;
+    else
+        totals.turns_stopped += 1;
 }
 function sumTotals(a, b) {
     const sum = emptyTotals();
@@ -181,6 +190,32 @@ export function startRunReport(options) {
             list,
         };
     };
+    /** The spec gaps the pulses' agents met, in the order the pulses started: questions a person would want to decide. */
+    const specGapsOf = (jobs) => jobs.flatMap((job) => {
+        const details = job.details;
+        return (details?.specGaps ?? []).map((gap) => ({
+            module: job.module ?? null,
+            pulse: job.pulse ?? null,
+            pulse_id: job.id,
+            landed: job.status === "succeeded" && details?.landed === true,
+            question: gap.question,
+            assumption: gap.assumption,
+        }));
+    });
+    /** Each request the provider failed and was waited out or stalled on, and each turn turri ended itself: why, and where. */
+    const turnIncidentsOf = () => events
+        .filter((event) => event.kind === "provider_retry" || event.kind === "provider_stall" || event.kind === "stopped")
+        .map((event) => ({
+        kind: event.kind,
+        pulse_id: event.job ?? null,
+        role: event.role ?? null,
+        model: event.model ?? null,
+        ...(event.kind === "stopped"
+            ? { reason: event.reason ?? null }
+            : event.kind === "provider_stall"
+                ? { silent_seconds: Math.round((event.waitMs ?? 0) / 1000) }
+                : { error: event.error ?? null, attempt: event.attempt ?? null, of: event.of ?? null, wait_seconds: Math.round((event.waitMs ?? 0) / 100) / 10 }),
+    }));
     const functionalitiesOf = (counts) => {
         if (counts === undefined)
             return null;
@@ -228,6 +263,7 @@ export function startRunReport(options) {
             build_dir: options.buildDir !== undefined ? resolve(options.buildDir) : null,
             conformance_tests: options.conformanceTests,
             review: options.review ?? null,
+            mutation_testing: options.mutationTesting ?? null,
             models: {
                 ...Object.fromEntries(Object.entries(options.models).map(([role, choice]) => [role, { model: choice.model ?? null, thinking: choice.thinking ?? null }])),
                 observed: Object.fromEntries([...new Set(events.map((event) => event.role ?? "unknown"))].sort().map((role) => [role, [...new Set(events.filter((event) => (event.role ?? "unknown") === role).map((event) => event.model ?? "unknown"))].sort()])),
@@ -245,6 +281,8 @@ export function startRunReport(options) {
             usage: usageOf(jobs),
             pulses: pulsesOf(jobs, now),
             functionalities: functionalitiesOf(attempt(sources.completion)),
+            spec_gaps: specGapsOf(jobs),
+            turn_incidents: turnIncidentsOf(),
             ...(exitParts ?? {}),
             logs: {
                 jobs_dir: fromProject(join(projectRoot, ".turri", "jobs")),

package/docs/config.md

@@ -84,7 +84,7 @@ runs:
 | --- | --- |
 | now | `harness-model` and `models.chat` - the conversation switches to it; `parallel-jobs` - a pulse waiting for a slot starts if there is one now; `allow-to-stop-jobs` |
 | your next message | `rules-off` - for the conversation; a skill's agents from the next skill run |
-| the next pulse - one running keeps what it started with | `coding-model`, `judge-model`, every other `models.<role>`, `review`, `turn-limit`, `compact-at`, `agent-env`, `toolchains` |
+| the next pulse - one running keeps what it started with | `coding-model`, `judge-model`, every other `models.<role>`, `review`, `mutation-testing`, `turn-limit`, `compact-at`, `agent-env`, `toolchains` |
 | the next time the work is planned | `conformance-tests`, `conformance-budget`, `max-specs-per-pulse`, `max-lines-per-pulse` |
 | the next test run - the running pulse's too | the testing scripts, `testing-scripts-run-on` |
 | the next pulse that lands - the running one too | `release-after-pulse` |
@@ -128,6 +128,7 @@ models:                              # each agent role's model, thinking and tur
 | `conformance-tests` | `all` (or `true`): conformance tests for every functional spec of every module with test reqs, written once each module's code is built, before anything is built on it. `chosen`: for the functional specs chosen - every one with acceptance tests, and those the chooser picks as each code pulse ends, from what that pulse regenerated. `module`: the same, but the chooser is asked once, when the whole module is regenerated, and plans the conformance pulses too. Under both, the choice is made inside the pulse, before anything is built on the module, and the pulse's report says what was chosen and why; those the person asks for with `conform` are written too; once chosen, a functional spec keeps them, and they are rewritten when it changes - see [who chooses, and when](regenerating.md#conformance-tests-in-the-work-loop). `false`: none written, and none of the existing ones run - until the conversation asks with `conform` | `false` |
 | `conformance-budget` | under `conformance-tests: chosen` or `module`: how many of a module's functional specs may be chosen for conformance tests, beside those with acceptance tests - a count (`2`) or a share of its functional specs (`30%`, rounded up, at least one). Every choice for the module counts, whenever it was made; the chooser is told what is left, and can choose no more. Your own `conform` is not limited. A stack's `config.turri.<stack>.yaml` can set its own | no limit |
 | `review` | `true` or `false`: whether the code and its unit tests - and the conformance tests - are reviewed once they pass: two reviews at most, the second weighed by the arbiter; off, a job lands once its tests pass - see [regenerating the code](regenerating.md#the-code-and-its-unit-tests-are-reviewed) | `false` |
+| `mutation-testing` | `true` or `false`: whether the coding agent and the reviewer have `can_it_fail` - one change put in the code for one run of the unit testing script, to see that a test catches it; on a copy of the code, or in place and put back after the run (see [regenerating the code](regenerating.md#checking-its-own-work)). `turri regenerate --mutation-testing false` for one run | `true` |
 | `rules-off` | a list of rule names - `integrations`, `concise-specs`, the files of `genome/rules/` without `.md` - left out of the conversation's prompt and every spec-writing agent's: a rule this project's own skills or AGENTS.md replace, such as one that says to probe a live API before any spec is written. The project's alone | none |
 | `allow-to-stop-jobs` | `true` or `false`: whether turri stops a running job the spec has moved on from - a functional spec it regenerates changed, or a file one links was edited - rolls it back and regenerates the spec as it now is. `false`: the job runs to its end and lands, and what changed since is regenerated on top of it by the next job - see [regenerating the code](regenerating.md). Your own `halt` stops a job either way | `true` |
 | `release-after-pulse` | `true` or `false`: whether every pulse that lands code releases it, in the chat, as its last stage - before the module's next pulse starts, whatever the chat is doing: an agent on the coding model looks for the project's release guidance as the chat does (what was passed to `release` this conversation, AGENTS.md, the project's skills) and, when it says to release after a build, does what it says - see [deploy](deploy.md#after-every-pulse). One release a pulse that lands; a release that fails is said in the pulse's report, and what it landed stays | `false` |
@@ -142,7 +143,7 @@ models:                              # each agent role's model, thinking and tur
 | `toolchains` | folders of this machine an agent's commands may read beside the system's, the `PATH`'s and those the toolchain variables point at - a toolchain installed somewhere unusual (`[/srv/zig]`). A stack's add to the project's. See [the OS sandbox](sandbox.md#the-os-sandbox) | none |
 | `turn-limit` | seconds each turn of an agent a job starts has, for every role without its own (`models:`); `0` for none. The config is the only place it is set. A turn past it ends at its next request to the model - see [regenerating the code](regenerating.md) | 120 |
 | `models` | each agent role's own model, thinking and turn limit - see [models per role](#models-per-role) | every role on `coding-model`, the conversation's side on `harness-model` |
-| `testing-scripts-run-on` | what the testing scripts run on - always on this machine, in your environment. `in-place`: the real build and tests folders, the script at its own path, from the project - so a script that finds a host codebase from its own path finds it. `copy`: a fresh copy of the code, staged in `plain_modules/.turri/scripts/`, with the script copied beside it; it gives the coding agent `can_it_fail` (see [the testing scripts](regenerating.md#the-testing-scripts)) | `in-place` |
+| `testing-scripts-run-on` | what the testing scripts run on - always on this machine, in your environment. `in-place`: the real build and tests folders, the script at its own path, from the project - so a script that finds a host codebase from its own path finds it. `copy`: a fresh copy of the code, staged in `plain_modules/.turri/scripts/`, with the script copied beside it. `can_it_fail` works on either: on a copy it changes the copy, in place the code itself for the one run, put back after it (see [the testing scripts](regenerating.md#the-testing-scripts)) | `in-place` |
 
 Keys are written with dashes; underscores are read the same (`conformance_tests`). A key turri
 does not know is ignored - another tool's `config.yaml`, read while it is the old file, may carry several - and so is a value of the