@turri-dev/cli 0.1.3-next.f74e98a → 0.1.4-next.172be6f

package/dist/regenerate/regression.js

@@ -21,7 +21,7 @@ import { keepingTestsUnchanged, listFiles } from "./testing.js";
  * job ends on it: tests that pass on their own are not done until the whole suite passes with them.
  */
 /** On top of the file tools, which are fenced to the build directory by `confine.ts`. */
-const FIX_TOOLS = ["run_unit_tests", "run_conformance_tests", "update_notes"];
+const FIX_TOOLS = ["run_unit_tests", "run_conformance_tests", "update_notes", "report_spec_gap"];
 const DEFAULT_MAX_REGRESSION_ROUNDS = ATTEMPTS.regression;
 /** The module's conformance tests, when there are any: the chain's, copied up, and its own. */
 function conformanceDirs(conformanceRoot, moduleName) {
@@ -39,6 +39,7 @@ async function repairRegression(input) {
         cwd: input.buildDir,
         customTools: [
             notes.tool,
+            notes.gapTool,
             ...createWebTools(),
             ...fence.tools,
             createScriptTool({ kind: "unit", script: input.scripts.unit, run: () => runUnitScript(input.buildDir, input.scripts.unit), runs }),

package/dist/regenerate/requires.js

@@ -1,6 +1,6 @@
-import { copyFileSync, existsSync, mkdirSync, readdirSync, rmSync } from "node:fs";
+import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs";
 import { basename, dirname, join, resolve } from "node:path";
-import { PLAIN_SOURCE_FILE_EXTENSION, plainFileParser, withParseCache } from "../plain/parser.js";
+import { IMPORT_DIRECTIVE, PLAIN_SOURCE_FILE_EXTENSION, plainFileParser, readPlainSourceMetadata, withParseCache } from "../plain/parser.js";
 import { getFrids, getInheritedFunctionalSpecs } from "../plain/spec.js";
 import { buildDirUnder, modulesRoot, projectDirOf } from "./paths.js";
 import { notesPath } from "./notes.js";
@@ -54,6 +54,59 @@ options = {}) {
     visit(resolve(plainFile), []);
     return chain;
 }
+/** The .plain files that import `plainFile` - read from beside it, and, kept in a `template/` folder, from the folder it is in. */
+function importersOf(plainFile) {
+    const file = resolve(plainFile);
+    const dir = dirname(file);
+    const name = basename(file, PLAIN_SOURCE_FILE_EXTENSION);
+    const dirs = [dir, ...(basename(dir) === "template" ? [dirname(dir)] : [])];
+    return dirs.flatMap((at) => {
+        let names;
+        try {
+            names = readdirSync(at).filter((entry) => entry.endsWith(PLAIN_SOURCE_FILE_EXTENSION));
+        }
+        catch {
+            return [];
+        }
+        return names
+            .map((entry) => join(at, entry))
+            .filter((path) => {
+            if (path === file || !resolveTemplateDirs(path).includes(dir))
+                return false;
+            try {
+                return (readPlainSourceMetadata(readFileSync(path, "utf8")).metadata[IMPORT_DIRECTIVE] ?? []).includes(name);
+            }
+            catch {
+                return false;
+            }
+        });
+    });
+}
+/**
+ * Whether a .plain file is an import module - one a .plain file imports, or one kept in a `template/` folder - rather than
+ * a module of its own. It is no codebase: its reqs and definitions are built in the modules that import it.
+ */
+export function isImportModule(plainFile) {
+    return basename(dirname(resolve(plainFile))) === "template" || importersOf(plainFile).length > 0;
+}
+/** The modules of their own that import an import module, through any import modules between. */
+export function importingModules(plainFile) {
+    const found = new Set();
+    const seen = new Set();
+    const walk = (file) => {
+        if (seen.has(file))
+            return;
+        seen.add(file);
+        for (const importer of importersOf(file)) {
+            if (isImportModule(importer))
+                walk(importer);
+            else
+                found.add(importer);
+        }
+    };
+    walk(resolve(plainFile));
+    return [...found];
+}
 /**
  * The codebase a change to `plainFile`'s module lands in: the furthest-built module whose
  * requires chain includes it, with that chain.

package/dist/regenerate/self-check.js

@@ -1,9 +1,9 @@
-import { readFileSync, writeFileSync } from "node:fs";
-import { join, relative, resolve, sep } from "node:path";
+import { readFileSync } from "node:fs";
+import { relative, resolve, sep } from "node:path";
 import { defineTool } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
 import { workDiff } from "./lineage.js";
-import { lastPassingMs, runOnCode } from "./test-scripts.js";
+import { lastPassingMs, runOnCode, scriptsRunInPlace } from "./test-scripts.js";
 /**
  * Tools an agent checks its own work with, before the harness and the reviewer do: the diff the reviewer is shown, and
  * whether a test catches a change to the code. Neither decides anything - the testing scripts, run by the harness, still do.
@@ -43,16 +43,23 @@ export function canItFailLimitMs(buildDir, script) {
  */
 export function createCanItFailTool(options) {
     let left = options.runs ?? CAN_IT_FAIL_RUNS;
+    // In place, the change is made to the code itself for the run, and undone after it: the scripts run on the code there.
+    const inPlace = scriptsRunInPlace(options.buildDir);
     return defineTool({
         name: "can_it_fail",
         label: "Can it fail",
         description: "Check that the unit tests catch a bug: give one change to the code that breaks a behaviour the specification " +
             "states - a value or a status changed, a check taken out, a condition turned round where it cannot make a loop run for " +
             "ever - and the unit testing script " +
-            "runs on a copy of the code with that change. The code itself is never touched. The tests should fail; if they all " +
+            (inPlace
+                ? "runs with that change made to the code, which is put back as it was once the run ends, whatever its result. "
+                : "runs on a copy of the code with that change. The code itself is never touched. ") +
+            "The tests should fail; if they all " +
             "pass, no test catches that bug. Make a change that still builds and runs: a failure only because it no longer " +
             `compiles or imports says nothing about the tests. ${options.runs ?? CAN_IT_FAIL_RUNS} runs at most.`,
-        promptSnippet: "can_it_fail: whether the unit tests catch a bug you put in a copy of the code",
+        promptSnippet: "can_it_fail: whether the unit tests catch a bug you put in the code for one run",
+        // Never beside another call: in place, an edit made while the bug is in would be undone with it.
+        executionMode: "sequential",
         parameters: Type.Object({
             file: Type.String({ description: "The code file to change, relative to the code folder" }),
             find: Type.String({ description: "Text that is in the file exactly once - the code to change" }),
@@ -83,7 +90,7 @@ export function createCanItFailTool(options) {
             left -= 1;
             let run;
             try {
-                run = await runOnCode(options.buildDir, options.script, "unit testing script, with the change", (copy) => writeFileSync(join(copy, relative(options.buildDir, path)), text.slice(0, at) + replace + text.slice(at + find.length)), canItFailLimitMs(options.buildDir, options.script));
+                run = await runOnCode(options.buildDir, options.script, "unit testing script, with the change", { file: relative(options.buildDir, path), text: text.slice(0, at) + replace + text.slice(at + find.length) }, canItFailLimitMs(options.buildDir, options.script));
             }
             catch (error) {
                 return say(error instanceof Error ? error.message : String(error));

package/dist/regenerate/test-scripts.js

@@ -297,6 +297,10 @@ export function setScriptTimeoutForTests(ms) {
  * own paths start - and whether they run on the code in place (the default), rather than on a fresh copy
  * (`testing-scripts-run-on: copy`).
  */
+/** Whether the testing scripts run on the code in `buildDir` itself, not on a copy of it. */
+export function scriptsRunInPlace(buildDir) {
+    return projectOfBuild(buildDir).inPlace;
+}
 function projectOfBuild(buildDir) {
     const parts = resolve(buildDir).split(sep);
     const index = parts.lastIndexOf("plain_modules");
@@ -401,30 +405,70 @@ const lastPassing = new Map();
 export function lastPassingMs(buildDir, script) {
     return lastPassing.get(`${resolve(buildDir)}\n${script}`);
 }
+/** Files changed in place for a run under way, and what each held: put back however the run ends - the process exiting too. */
+const changedInPlace = new Map();
+let restoreOnExit = false;
+function putBack(path) {
+    const original = changedInPlace.get(path);
+    if (original === undefined)
+        return;
+    writeFileSync(path, original);
+    changedInPlace.delete(path);
+}
+/**
+ * `run` with `change` made to the code itself, and undone once it ends - passed, failed, timed out, thrown, or the process
+ * exiting. Killed outright, the change is left: the next unit of work starts from the last commit (lineage.ts startWork).
+ */
+async function withChangeInPlace(buildDir, change, run) {
+    const path = resolve(buildDir, change.file);
+    if (changedInPlace.has(path))
+        throw new Error(`${change.file} is being changed for another run already.`);
+    if (!restoreOnExit) {
+        restoreOnExit = true;
+        process.once("exit", () => {
+            for (const changed of [...changedInPlace.keys()])
+                putBack(changed);
+        });
+    }
+    changedInPlace.set(path, readFileSync(path));
+    try {
+        writeFileSync(path, change.text);
+        return await run();
+    }
+    finally {
+        putBack(path);
+    }
+}
 /**
- * A script that takes the code folder - the unit testing script - run where the testing scripts
- * run. `change`: what to do to the fresh copy before it runs - only where there is a copy; never in place.
+ * A script that takes the code folder - the unit testing script - run where the testing scripts run. `change`: one file
+ * as it is for this run - made to the fresh copy, or, in place, to the code itself and undone once the run ends.
  */
 export async function runOnCode(buildDir, script, what, change, 
 /** A time limit of its own, in place of a test run's. */
 limitMs) {
     const project = projectOfBuild(buildDir);
+    const passed = (run, started) => {
+        if (change === undefined && run.exitCode === 0)
+            lastPassing.set(`${resolve(buildDir)}\n${script}`, Date.now() - started);
+        return run;
+    };
     if (project.inPlace) {
-        if (change !== undefined)
-            throw new Error("This runs only where the testing scripts run on a copy: here they run in place, on the code itself.");
-        return runScript(project.root, script, [resolve(buildDir)], buildDir);
+        const inPlace = () => runScript(project.root, script, [resolve(buildDir)], buildDir, limitMs !== undefined ? { limitMs } : {});
+        const started = Date.now();
+        if (change === undefined)
+            return passed(await inPlace(), started);
+        reportActivity(`  changing ${change.file} in place for the ${what} - put back once it has run`);
+        return withChangeInPlace(buildDir, change, inPlace);
     }
     const staging = stagingFor(buildDir);
     rmSync(staging.root, { recursive: true, force: true });
     ensureInModulesScratch(buildDir, staging.root);
     try {
         reportActivity(`  copying ${stageBuild(buildDir, staging.build)} files of the code for the ${what}`);
-        change?.(staging.build);
+        if (change !== undefined)
+            writeFileSync(join(staging.build, change.file), change.text);
         const started = Date.now();
-        const run = await runOnCopy(staging.root, script, [staging.build], buildDir, limitMs);
-        if (change === undefined && run.exitCode === 0)
-            lastPassing.set(`${resolve(buildDir)}\n${script}`, Date.now() - started);
-        return run;
+        return passed(await runOnCopy(staging.root, script, [staging.build], buildDir, limitMs), started);
     }
     finally {
         rmSync(staging.root, { recursive: true, force: true });

package/dist/regenerate/unit-review.js

@@ -63,6 +63,17 @@ export function unitReviewTask(options) {
         options.spec.trim(),
         "</specification>",
         "",
+        ...(options.testingScript !== undefined
+            ? [
+                "The harness runs the unit tests with this script, and it decides whether they pass. Read it for which tests it",
+                "runs and how: a test it never reaches - a file it does not pick up, a folder it does not look in - is not run, and",
+                "checks nothing.",
+                `<unit testing script path="${options.testingScript.path}">`,
+                options.testingScript.content.trimEnd(),
+                "</unit testing script>",
+                "",
+            ]
+            : []),
         ...(options.cases !== undefined ? [...codeReviewCasesBlock(options.cases), ""] : []),
         // A job that touches no functional spec is held only to that: the checks of behaviour are not its.
         ...(mechanical ? [] : ["Check that the code does what the specification states for this work:"]),
@@ -85,7 +96,7 @@ export function unitReviewTask(options) {
                 "  assertion so loose any answer passes, an expected value computed the way the code computes it.",
                 ...(options.canItFail === true
                     ? [
-                        "- A test you doubt can fail, check: can_it_fail puts a bug in a copy of the code - a condition turned round, a",
+                        "- A test you doubt can fail, check: can_it_fail puts a bug in the code for one run - a condition turned round, a",
                         "  value changed - and runs the unit tests on it. A test that still passes cannot fail: raise it as bad, with the",
                         `  bug it missed. ${REVIEWER_CAN_IT_FAIL_RUNS} runs a session: spend them on the tests the work rests on.`,
                     ]

package/dist/regenerate-cli.js

@@ -19,10 +19,15 @@ its conformance tests too when they are on.
   --start-over             start over, without asking, a codebase regenerated without a module it now requires -
                            with no terminal to ask and without it, the run stops there
   --from-scratch           delete the module's code and conformance tests (its lineage keeps them) and regenerate all of it
+  --from-scratch-chain     the same for every module of its requires chain, built again from the bottom - after
+                           modules were split or moved
   --harness-model <p/id>   config.turri.yaml's harness-model for this run: the planner and the conformance chooser
   --coding-model <p/id>    its coding-model, --model too: the agents that build; p/id:level for how hard a model thinks
   --judge-model <p/id>     its judge-model: the judge and the test review
   --review <true|false>    its review: whether the code and its tests are reviewed once they pass
+  --mutation-testing <true|false>
+                           its mutation-testing: whether the coding agent and the reviewer can put a bug in the code
+                           for one test run (can_it_fail), to see that a test catches it
   --conformance-tests <all|chosen|module|false>
                            its conformance-tests: which conformance tests are written and run
   --build-dir <dir>        as the run ends, however it ends, export the code there: the furthest-built codebase's
@@ -31,7 +36,7 @@ its conformance tests too when they are on.
                            as it starts, as each pulse ends and as it exits (docs/headless.md)
 
   PLAIN_TEMPLATE_DIRS      colon-separated template directories`;
-const { positional, options } = parseArgs(process.argv.slice(2), ["--dry-run", "--start-over", "--from-scratch"], USAGE, ["--build-dir", "--report", "--conformance-tests", "--harness-model", "--judge-model", "--review"]);
+const { positional, options } = parseArgs(process.argv.slice(2), ["--dry-run", "--start-over", "--from-scratch", "--from-scratch-chain"], USAGE, ["--build-dir", "--report", "--conformance-tests", "--harness-model", "--judge-model", "--review", "--mutation-testing"]);
 const { report, buildDir } = headlessOptions(options, true);
 const named = positional[0] ?? readProjectConfig(process.cwd()).module;
 // Functional specs named after the file: regenerated again though their spec did not change.
@@ -40,9 +45,10 @@ if (!named) {
     console.error(`${USAGE}\n\nNo file given, and ${CONFIG_FILE} names no module.`);
     process.exit(1);
 }
+const scratch = options["from-scratch-chain"] !== undefined ? "chain" : options["from-scratch"] !== undefined ? "module" : undefined;
 // A dry run starts nothing, and from scratch deletes before it plans: the two do not go together.
-if (options["from-scratch"] !== undefined && options["dry-run"] !== undefined) {
-    console.error("--from-scratch deletes the module's code before anything is planned: it cannot be a dry run.");
+if (scratch !== undefined && options["dry-run"] !== undefined) {
+    console.error(`--from-scratch${scratch === "chain" ? "-chain" : ""} deletes the code before anything is planned: it cannot be a dry run.`);
     process.exit(1);
 }
 runLoop({
@@ -54,8 +60,8 @@ runLoop({
     startOver: options["start-over"] !== undefined,
     ask: (loop) => again.length > 0
         ? loop.regenerateAgain(named, again)
-        : options["from-scratch"] !== undefined
-            ? loop.fromScratch(named)
+        : scratch !== undefined
+            ? loop.fromScratch(named, { chain: scratch === "chain" })
             : loop.bringUp(named),
 }).catch((error) => {
     console.error(error instanceof Error ? error.message : error);

package/dist/release-tools.js

@@ -142,7 +142,7 @@ export function createDeployTools(options) {
         },
     });
     // The one halt: with the pulses in the conversation, and without them for the release agent.
-    const stopTool = haltTool(options.jobs !== undefined ? { jobs: options.jobs } : {});
+    const stopTool = haltTool(options.jobs !== undefined ? { jobs: options.jobs, ...(options.workLoop !== undefined ? { workLoop: options.workLoop } : {}) } : {});
     return [deploy, startTool, stopTool];
 }
 export const DEPLOY_TOOLS = ["release", "activate", "halt"];

package/dist/resume-line.js

@@ -0,0 +1,4 @@
+/** pi's line on the way out that says how to resume the session, with turri in place of pi as the command to run. */
+export function resumeLine(text) {
+    return text.replace(/^((?:\x1b\[[0-9;]*m)*To resume this session:(?:\x1b\[[0-9;]*m)*\s+)pi(?= )/m, "$1turri");
+}

package/dist/review/live.js

@@ -79,7 +79,8 @@ export function createLiveReviewer(opts) {
             pending.push(finding);
             all.push(finding);
         }
-        if (violations.length === 0) {
+        // No verdict - its turn stopped before it gave one - clears nothing: the scope is looked at again next pass.
+        if (captured !== undefined && violations.length === 0) {
             cleared.push(scope);
         }
         else if (captured?.question) {

package/dist/run-report.js

@@ -22,6 +22,8 @@ const emptyTotals = () => ({
     llm_calls: 0,
     agent_turns: 0,
     compactions: 0,
+    provider_retries: 0,
+    turns_stopped: 0,
     model_seconds: 0,
 });
 function addUsage(totals, usage) {
@@ -43,8 +45,12 @@ function addEvent(totals, event, tokens) {
     }
     else if (event.kind === "turn")
         totals.agent_turns += 1;
-    else
+    else if (event.kind === "compaction")
         totals.compactions += 1;
+    else if (event.kind === "provider_retry")
+        totals.provider_retries += 1;
+    else
+        totals.turns_stopped += 1;
 }
 function sumTotals(a, b) {
     const sum = emptyTotals();
@@ -181,6 +187,30 @@ export function startRunReport(options) {
             list,
         };
     };
+    /** The spec gaps the pulses' agents met, in the order the pulses started: questions a person would want to decide. */
+    const specGapsOf = (jobs) => jobs.flatMap((job) => {
+        const details = job.details;
+        return (details?.specGaps ?? []).map((gap) => ({
+            module: job.module ?? null,
+            pulse: job.pulse ?? null,
+            pulse_id: job.id,
+            landed: job.status === "succeeded" && details?.landed === true,
+            question: gap.question,
+            assumption: gap.assumption,
+        }));
+    });
+    /** Each turn the provider failed and turri waited out, and each turn turri ended itself: why, and where. */
+    const turnIncidentsOf = () => events
+        .filter((event) => event.kind === "provider_retry" || event.kind === "stopped")
+        .map((event) => ({
+        kind: event.kind,
+        pulse_id: event.job ?? null,
+        role: event.role ?? null,
+        model: event.model ?? null,
+        ...(event.kind === "stopped"
+            ? { reason: event.reason ?? null }
+            : { error: event.error ?? null, attempt: event.attempt ?? null, of: event.of ?? null, wait_seconds: Math.round((event.waitMs ?? 0) / 100) / 10 }),
+    }));
     const functionalitiesOf = (counts) => {
         if (counts === undefined)
             return null;
@@ -228,6 +258,7 @@ export function startRunReport(options) {
             build_dir: options.buildDir !== undefined ? resolve(options.buildDir) : null,
             conformance_tests: options.conformanceTests,
             review: options.review ?? null,
+            mutation_testing: options.mutationTesting ?? null,
             models: {
                 ...Object.fromEntries(Object.entries(options.models).map(([role, choice]) => [role, { model: choice.model ?? null, thinking: choice.thinking ?? null }])),
                 observed: Object.fromEntries([...new Set(events.map((event) => event.role ?? "unknown"))].sort().map((role) => [role, [...new Set(events.filter((event) => (event.role ?? "unknown") === role).map((event) => event.model ?? "unknown"))].sort()])),
@@ -245,6 +276,8 @@ export function startRunReport(options) {
             usage: usageOf(jobs),
             pulses: pulsesOf(jobs, now),
             functionalities: functionalitiesOf(attempt(sources.completion)),
+            spec_gaps: specGapsOf(jobs),
+            turn_incidents: turnIncidentsOf(),
             ...(exitParts ?? {}),
             logs: {
                 jobs_dir: fromProject(join(projectRoot, ".turri", "jobs")),

package/dist/spec-tools.js

@@ -1,8 +1,8 @@
-import { readdirSync, readFileSync } from "node:fs";
-import { basename, dirname, join, relative, resolve } from "node:path";
+import { basename, relative, resolve } from "node:path";
 import { defineTool } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
-import { IMPORT_DIRECTIVE, PLAIN_SOURCE_FILE_EXTENSION, plainFileParser, readPlainSourceMetadata } from "./plain/parser.js";
+import { plainFileParser } from "./plain/parser.js";
+import { importingModules, isImportModule } from "./regenerate/requires.js";
 import { resolveTemplateDirs } from "./regenerate/paths.js";
 import { applySpecDiff, plainFilesOf, revertSpecDiff } from "./regenerate/regenerate.js";
 import { MissingFunctionalitiesError } from "./plain/errors.js";
@@ -62,24 +62,12 @@ export function createSpecDiffTools(options) {
             // file that cannot be parsed. A module with no functional specs yet is not broken. An import
             // module - definitions only, say - is not a module of its own: it is parsed as the modules
             // that import it read it, the one the change is built for among them.
-            const importers = (file) => {
-                const name = basename(file, ".plain");
-                const dir = dirname(resolve(cwd, file));
-                return readdirSync(dir)
-                    .filter((entry) => entry.endsWith(".plain"))
-                    .map((entry) => join(dir, entry))
-                    .filter((path) => {
-                    try {
-                        return (readPlainSourceMetadata(readFileSync(path, "utf8")).metadata[IMPORT_DIRECTIVE] ?? []).includes(name);
-                    }
-                    catch {
-                        return false;
-                    }
-                });
-            };
-            const imported = importedNames(resolveTemplateDirs(resolve(cwd, params.plain_file)));
             const toParse = [
-                ...new Set([params.plain_file, ...touched].flatMap((file) => imported.has(basename(file, ".plain")) ? importers(file).filter((path) => !imported.has(basename(path, ".plain"))) : [resolve(cwd, file)])),
+                ...new Set([params.plain_file, ...touched].flatMap((file) => {
+                    const path = resolve(cwd, file);
+                    const importing = isImportModule(path) ? importingModules(path) : [];
+                    return importing.length > 0 ? importing : [path];
+                })),
             ];
             for (const path of toParse) {
                 const file = relative(cwd, path);
@@ -100,7 +88,8 @@ export function createSpecDiffTools(options) {
             owed = false;
             liveReview?.notice(`${params.plain_file}, after the change to ${touched.join(", ")}`);
             // In the files, the change is work: the loop works out what is left to build and plans it.
-            const statuses = await loop.bringUp(params.plain_file, params.guidance !== undefined ? { guidance: params.guidance } : {});
+            // A spec change, not a request: a module the person halted stays halted.
+            const statuses = await loop.bringUp(params.plain_file, { specChange: true, ...(params.guidance !== undefined ? { guidance: params.guidance } : {}) });
             const lines = [
                 result === "applied" ? `Applied the change to ${touched.join(", ")}.` : `The change to ${touched.join(", ")} was already applied.`,
                 `Building in the background:\n${describeModules(statuses)}\n`,
@@ -129,27 +118,3 @@ export function createSpecDiffTools(options) {
         },
     };
 }
-/** The module names some .plain file in these folders imports: import modules, not codebases. */
-export function importedNames(dirs) {
-    const names = new Set();
-    for (const dir of dirs) {
-        let entries;
-        try {
-            entries = readdirSync(dir).filter((name) => name.endsWith(PLAIN_SOURCE_FILE_EXTENSION));
-        }
-        catch {
-            continue;
-        }
-        for (const entry of entries) {
-            try {
-                const imports = readPlainSourceMetadata(readFileSync(join(dir, entry), "utf8")).metadata[IMPORT_DIRECTIVE];
-                for (const name of imports ?? [])
-                    names.add(name);
-            }
-            catch {
-                // A file that does not read imports nothing.
-            }
-        }
-    }
-    return names;
-}

package/docs/authoring-plain.md

@@ -196,18 +196,21 @@ the reading that fits the rest of the spec, and marks each such decision in its
 code, environment and turri problems to the builds, and stops when a functionality keeps
 struggling after the spec was clarified, when two cycles change nothing, or after twenty.
 
-For an existing project, `bud` works the same way, scoped to one feature: intent → scope →
-product → review. After a short feature-intent interview it asks product questions one at a time -
-a trigger and its outcome, an edge case, a rule - and every answer is one spec change, a git diff
-written with `write_spec_diff` and built the moment it is written. A change to something already
+For an existing project, `bud` works the same way, scoped to one feature: intent → scope → product.
+It first asks how many questions the product phase may ask - light, medium, thorough or best
+judgement, as `brood` does - then a short feature-intent interview, then product questions - a
+trigger and its outcome, an edge case, a rule - and every answer is a spec change, a git diff
+written with `write_spec_diff` and built the moment it is written, before the next question. What
+the questions leave open it decides itself, a spec change each. A change to something already
 specified, built or still to build, is put to the user first and is a spec diff too: what is left to
-build is worked out again from the spec.
+build is worked out again from the spec. It ends on the person's confirmation of a short summary of
+the feature's changes.
 
 What keeps the interview honest is the harness, not the interviewer:
 
 - **Every question offers choices** (`ask_user` refuses a question without them), and each
   answer maps to one spec change.
-- **Every answer is written before the next question.** In `brood`'s product phase `ask_user` refuses
+- **Every answer is written before the next question.** In the product phase of `brood` and `bud`, `ask_user` refuses
   a new question until the last answer is a spec change - see [skills and gates](skills-and-gates.md).
 - **Each phase ends on the person's confirmation** of a short summary, not on the interviewer's word.
 

package/docs/config.md

@@ -84,7 +84,7 @@ runs:
 | --- | --- |
 | now | `harness-model` and `models.chat` - the conversation switches to it; `parallel-jobs` - a pulse waiting for a slot starts if there is one now; `allow-to-stop-jobs` |
 | your next message | `rules-off` - for the conversation; a skill's agents from the next skill run |
-| the next pulse - one running keeps what it started with | `coding-model`, `judge-model`, every other `models.<role>`, `review`, `turn-limit`, `compact-at`, `agent-env`, `toolchains` |
+| the next pulse - one running keeps what it started with | `coding-model`, `judge-model`, every other `models.<role>`, `review`, `mutation-testing`, `turn-limit`, `compact-at`, `agent-env`, `toolchains` |
 | the next time the work is planned | `conformance-tests`, `conformance-budget`, `max-specs-per-pulse`, `max-lines-per-pulse` |
 | the next test run - the running pulse's too | the testing scripts, `testing-scripts-run-on` |
 | the next pulse that lands - the running one too | `release-after-pulse` |
@@ -128,6 +128,7 @@ models:                              # each agent role's model, thinking and tur
 | `conformance-tests` | `all` (or `true`): conformance tests for every functional spec of every module with test reqs, written once each module's code is built, before anything is built on it. `chosen`: for the functional specs chosen - every one with acceptance tests, and those the chooser picks as each code pulse ends, from what that pulse regenerated. `module`: the same, but the chooser is asked once, when the whole module is regenerated, and plans the conformance pulses too. Under both, the choice is made inside the pulse, before anything is built on the module, and the pulse's report says what was chosen and why; those the person asks for with `conform` are written too; once chosen, a functional spec keeps them, and they are rewritten when it changes - see [who chooses, and when](regenerating.md#conformance-tests-in-the-work-loop). `false`: none written, and none of the existing ones run - until the conversation asks with `conform` | `false` |
 | `conformance-budget` | under `conformance-tests: chosen` or `module`: how many of a module's functional specs may be chosen for conformance tests, beside those with acceptance tests - a count (`2`) or a share of its functional specs (`30%`, rounded up, at least one). Every choice for the module counts, whenever it was made; the chooser is told what is left, and can choose no more. Your own `conform` is not limited. A stack's `config.turri.<stack>.yaml` can set its own | no limit |
 | `review` | `true` or `false`: whether the code and its unit tests - and the conformance tests - are reviewed once they pass: two reviews at most, the second weighed by the arbiter; off, a job lands once its tests pass - see [regenerating the code](regenerating.md#the-code-and-its-unit-tests-are-reviewed) | `false` |
+| `mutation-testing` | `true` or `false`: whether the coding agent and the reviewer have `can_it_fail` - one change put in the code for one run of the unit testing script, to see that a test catches it; on a copy of the code, or in place and put back after the run (see [regenerating the code](regenerating.md#checking-its-own-work)). `turri regenerate --mutation-testing false` for one run | `true` |
 | `rules-off` | a list of rule names - `integrations`, `concise-specs`, the files of `genome/rules/` without `.md` - left out of the conversation's prompt and every spec-writing agent's: a rule this project's own skills or AGENTS.md replace, such as one that says to probe a live API before any spec is written. The project's alone | none |
 | `allow-to-stop-jobs` | `true` or `false`: whether turri stops a running job the spec has moved on from - a functional spec it regenerates changed, or a file one links was edited - rolls it back and regenerates the spec as it now is. `false`: the job runs to its end and lands, and what changed since is regenerated on top of it by the next job - see [regenerating the code](regenerating.md). Your own `halt` stops a job either way | `true` |
 | `release-after-pulse` | `true` or `false`: whether every pulse that lands code releases it, in the chat, as its last stage - before the module's next pulse starts, whatever the chat is doing: an agent on the coding model looks for the project's release guidance as the chat does (what was passed to `release` this conversation, AGENTS.md, the project's skills) and, when it says to release after a build, does what it says - see [deploy](deploy.md#after-every-pulse). One release a pulse that lands; a release that fails is said in the pulse's report, and what it landed stays | `false` |
@@ -142,7 +143,7 @@ models:                              # each agent role's model, thinking and tur
 | `toolchains` | folders of this machine an agent's commands may read beside the system's, the `PATH`'s and those the toolchain variables point at - a toolchain installed somewhere unusual (`[/srv/zig]`). A stack's add to the project's. See [the OS sandbox](sandbox.md#the-os-sandbox) | none |
 | `turn-limit` | seconds each turn of an agent a job starts has, for every role without its own (`models:`); `0` for none. The config is the only place it is set. A turn past it ends at its next request to the model - see [regenerating the code](regenerating.md) | 120 |
 | `models` | each agent role's own model, thinking and turn limit - see [models per role](#models-per-role) | every role on `coding-model`, the conversation's side on `harness-model` |
-| `testing-scripts-run-on` | what the testing scripts run on - always on this machine, in your environment. `in-place`: the real build and tests folders, the script at its own path, from the project - so a script that finds a host codebase from its own path finds it. `copy`: a fresh copy of the code, staged in `plain_modules/.turri/scripts/`, with the script copied beside it; it gives the coding agent `can_it_fail` (see [the testing scripts](regenerating.md#the-testing-scripts)) | `in-place` |
+| `testing-scripts-run-on` | what the testing scripts run on - always on this machine, in your environment. `in-place`: the real build and tests folders, the script at its own path, from the project - so a script that finds a host codebase from its own path finds it. `copy`: a fresh copy of the code, staged in `plain_modules/.turri/scripts/`, with the script copied beside it. `can_it_fail` works on either: on a copy it changes the copy, in place the code itself for the one run, put back after it (see [the testing scripts](regenerating.md#the-testing-scripts)) | `in-place` |
 
 Keys are written with dashes; underscores are read the same (`conformance_tests`). A key turri
 does not know is ignored - another tool's `config.yaml`, read while it is the old file, may carry several - and so is a value of the

package/docs/headless.md

@@ -86,7 +86,10 @@ Useful options - `turri regenerate --help` lists all of them:
 | `--judge-model <provider/id>` | `judge-model`: the judge and the test review; not given, the coding model |
 | `--conformance-tests <all\|chosen\|module\|false>` | `conformance-tests`, with its values (`true` is `all`) |
 | `--review <true\|false>` | `review`: whether the code and its tests are reviewed once they pass |
+| `--mutation-testing <true\|false>` | `mutation-testing`: whether the coding agent and the reviewer have `can_it_fail` |
 | `--stack <name>` | build with the stack's `config.turri.<name>.yaml` over `config.turri.yaml`; default, `TURRI_STACK`, else none |
+| `--from-scratch` | delete the module's code and conformance tests (its lineage keeps them) and regenerate all of it |
+| `--from-scratch-chain` | the same for every module of its requires chain, the base's included, built again from the bottom |
 | `--start-over` | start over, unasked, a codebase built without a module it now requires - without a terminal, a run that needs it and lacks it stops |
 | `--build-dir <dir>` | as the run ends, export the code there - see below |
 | `--report <file.json>` | write [the run report](#the-run-report) there |
@@ -104,7 +107,8 @@ Conformance tests follow `conformance-tests:` in `config.turri.yaml`, or `--conf
 The command exits `0` only when every module is complete: its code regenerated from the spec as it is
 now, and its conformance tests written for what is built when they are on - no pulse failed, nothing
 halted, nothing waiting for a start over. Anything else exits `1`, and the last lines say which
-module is not complete, and why. The log says, per functionality, what was built and whether its
+module is not complete, and why. A behaviour the spec leaves open that an agent had to decide is printed
+as a `spec gap` under its pulse, with what was decided meanwhile. The log says, per functionality, what was built and whether its
 tests passed.
 
 | | |
@@ -150,6 +154,7 @@ changes meaning or goes; new fields may appear in the same version. Paths in `pu
   "spec_file": "/work/app.plain", "project_dir": "/work", "build_dir": "/work/out",  // build_dir: null without one
   "conformance_tests": "all",                                                       // all | chosen | module | false
   "review": false,
+  "mutation_testing": true,
   "models": {
     "harness": { "model": "…", "thinking": "medium" },  // the planner, the chooser
     "coding":  { "model": "…", "thinking": "low" },
@@ -165,7 +170,7 @@ changes meaning or goes; new fields may appear in the same version. Paths in `pu
   "usage": {
     "total": {
       "input_tokens": 0, "output_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cost_usd": 0,
-      "llm_calls": 0, "agent_turns": 0, "compactions": 0, "model_seconds": 0
+      "llm_calls": 0, "agent_turns": 0, "compactions": 0, "provider_retries": 0, "turns_stopped": 0, "model_seconds": 0
     },
     "in_pulses": { },       // the pulses' usage added up - the same fields
     "outside_pulses": { },  // the planner, the conformance chooser, a failed pulse's diagnosis
@@ -191,6 +196,15 @@ changes meaning or goes; new fields may appear in the same version. Paths in `pu
     ],
     "codebases": ["plain_modules/app/code"]
   },
+  "spec_gaps": [{                                       // [] when no agent met one
+    "module": "app", "pulse": 3, "pulse_id": "2a058729", "landed": true,
+    "question": "What happens to a contact archived in HubSpot after it was synced?",
+    "assumption": "An archived contact is left as synced - asserted in sync.test.ts, archived contacts"
+  }],
+  "turn_incidents": [                                   // [] when none
+    { "kind": "provider_retry", "pulse_id": "2a058729", "role": "coding", "model": "…", "error": "…: 529 overloaded", "attempt": 1, "of": 6, "wait_seconds": 15.3 },
+    { "kind": "stopped", "pulse_id": "5c1e0b2f", "role": "conformance explorer", "model": "…", "reason": "10 submit_exploration calls failed in a row" }
+  ],
   "build_dir_export": { "exported": true, "dir": "/work/out", "module": "app", "commit": "8036b3b", "files": 23 },
   "logs": { "jobs_dir": ".turri/jobs", "debug_dir": ".turri/debug", "sessions_dir": ".turri/sessions" }
 }
@@ -209,6 +223,19 @@ calls; a role or model turri could not tell is `unknown`.
 stopped, or did not pass - and everything it wrote was undone. A pulse that failed is tried once more
 before it counts as failed; both tries are the one pulse.
 
+**Spec gaps.** A behaviour the spec leaves open that an agent met and decided on its own - reported with
+`report_spec_gap` (see [regenerating](regenerating.md)). In the chat they are put to you as the pulse ends; a
+headless run has no one to ask, so they are listed here, in the order the pulses started, and printed under the
+pulse that met them. `assumption` is what the agent decided meanwhile, also in the codebase's notes'
+assumptions while its pulse is `landed`; a pulse rolled back takes it out of the notes, but the question still
+stands. Settle the ones you care about in the spec and run again.
+
+**Turn incidents.** Each time the model provider failed for a moment and turri waited it out before trying the
+turn again, and each turn turri stopped itself - an agent going round in a circle, its calls failing again and again,
+its time up - after which the run carried on. Both are printed as they happen, in yellow, under the pulse, and
+counted in the usage as `provider_retries` and `turns_stopped`. Many of either point at the model, not the spec: a
+provider at its limits, or a model too small for the role it was given.
+
 **Completed functional specs** - the pass rate, the figure to compare runs by.
 A functional spec is completed when the code record of the codebase the run delivers - the
 furthest-built of the chain, the one `--build-dir` exports - as last committed, holds it as the spec has
@@ -227,7 +254,7 @@ and modules that are complete are skipped, so it picks up where it stopped - aft
 provider outage, or a machine that went to sleep anyway. Whatever the stopped run left
 half-written is undone first: each step starts from the module's last commit. A functionality whose spec has changed
 since counts as not built, so editing the spec and re-running rebuilds what changed. `--from-scratch`
-rebuilds everything.
+rebuilds the module; `--from-scratch-chain` rebuilds every module of its chain.
 
 ## Rebuilding one functionality