@turri-dev/cli 0.1.8-next.e553476 → 0.1.9-next.73bfddf

package/dist/regenerate/lineage.js

@@ -2,6 +2,8 @@ import { spawnSync } from "node:child_process";
 import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { basename, dirname, join } from "node:path";
 import { reportActivity } from "./activity.js";
+import { historyError } from "./environment-error.js";
+import { firstLine, record } from "./ledger.js";
 import { projectRootOf } from "./paths.js";
 /**
  * Version history for each module's generated code and tests.
@@ -15,7 +17,8 @@ import { projectRootOf } from "./paths.js";
  * A commit is made after every step that changes the module - a render, a regeneration, a
  * conformance run, a regression repair, a spec change, a start from the module below - with
  * a message saying what it was. History is a record of the work, never a gate on it: a git
- * failure is swallowed, not raised.
+ * failure is swallowed, not raised - except landing work (`landWork`), whose commit the next
+ * start would otherwise undo.
  */
 export const HISTORY_DIR = ".plaingit";
 const AUTHOR = ["-c", "user.name=turri", "-c", "user.email=turri@localhost", "-c", "commit.gpgsign=false"];
@@ -39,7 +42,10 @@ const EXCLUDED = [
     ".DS_Store",
 ];
 function git(moduleDir, args, input) {
-    return spawnSync("git", [...AUTHOR, "--git-dir", join(moduleDir, HISTORY_DIR), "--work-tree", moduleDir, ...args], {
+    const gitDir = join(moduleDir, HISTORY_DIR);
+    // The history is turri's, not the person's: hooks their git config names - gitleaks, husky - would refuse some of its
+    // commits, and the next start would put back what landed. Its own hooks folder holds only git's samples.
+    return spawnSync("git", [...AUTHOR, "-c", `core.hooksPath=${join(gitDir, "hooks")}`, "--git-dir", gitDir, "--work-tree", moduleDir, ...args], {
         cwd: moduleDir,
         encoding: "utf8",
         input,
@@ -79,21 +85,65 @@ function ensureHistory(moduleDir) {
  */
 export function recordHistory(dir, subject, body) {
     try {
-        const moduleDir = moduleDirOf(dir);
-        if (!existsSync(moduleDir) || !ensureHistory(moduleDir))
-            return undefined;
-        git(moduleDir, ["add", "-A"]);
-        if (git(moduleDir, ["diff", "--cached", "--quiet"]).status === 0)
-            return undefined;
-        const message = body ? `${subject}\n\n${body.trim()}\n` : subject;
-        if (git(moduleDir, ["commit", "-q", "-F", "-"], message).status !== 0)
-            return undefined;
-        return head(dir);
+        return recorded(dir, subject, commitAll(dir, subject, body), false).commit;
     }
     catch {
         return undefined;
     }
 }
+/**
+ * Commit work that landed - the code or tests a pulse passed, with the record of what they are built from. Unlike any
+ * other step's, this commit is not only a record: the next unit of work puts the module back to its last commit, and
+ * would undo what landed with it. So a commit that cannot be made is raised, as the environment's, with what git said -
+ * the work left in place, uncommitted. Nothing changed, nothing is committed. Returns the commit, if any.
+ */
+export function landWork(dir, subject, body) {
+    const moduleDir = moduleDirOf(dir);
+    let made;
+    try {
+        made = commitAll(dir, subject, body);
+    }
+    catch (error) {
+        made = { failed: error instanceof Error ? error.message : String(error) };
+    }
+    recorded(dir, subject, made, true);
+    if (made.failed !== undefined)
+        throw historyError(moduleDir, made.failed);
+    return made.commit;
+}
+/** A commit made or failed, in the ledger - nothing for a step that changed nothing. */
+function recorded(dir, subject, made, landing) {
+    const place = placeOf(moduleDirOf(dir));
+    if (made.failed !== undefined)
+        record({ kind: "history.failed", subject: firstLine(subject, 200), said: firstLine(made.failed, 400), landing }, place);
+    else if (made.commit !== undefined)
+        record({ kind: "history.committed", subject: firstLine(subject, 200), commit: made.commit }, place);
+    return made;
+}
+/** Where a module's history is, for the ledger: its project and its name. */
+function placeOf(moduleDir) {
+    return { project: projectRootOf(moduleDir), module: basename(moduleDir) };
+}
+/** Commit everything the step changed in the module `dir` belongs to: the commit, nothing when nothing changed, or why git failed. */
+function commitAll(dir, subject, body) {
+    const moduleDir = moduleDirOf(dir);
+    if (!existsSync(moduleDir))
+        return {};
+    if (!ensureHistory(moduleDir))
+        return { failed: `git could not make the module's history in ${join(moduleDir, HISTORY_DIR)}` };
+    const failed = (result) => (result.error?.message ?? `${result.stderr}${result.stdout}`.trim()) || `git exited ${result.status}`;
+    const added = git(moduleDir, ["add", "-A"]);
+    if (added.status !== 0)
+        return { failed: failed(added) };
+    if (git(moduleDir, ["diff", "--cached", "--quiet"]).status === 0)
+        return {};
+    const message = body ? `${subject}\n\n${body.trim()}\n` : subject;
+    const committed = git(moduleDir, ["commit", "-q", "-F", "-"], message);
+    if (committed.status !== 0)
+        return { failed: failed(committed) };
+    const commit = head(dir);
+    return commit !== undefined ? { commit } : {};
+}
 /**
  * The notes the harness writes once a pulse has ended, committed - or the next pulse, putting the module back to its last
  * commit, would take them away. Into the pulse's own commit when it landed one - `landedSince` (ms), when it started - so
@@ -156,6 +206,7 @@ export function resetToLastCommit(dir) {
             return [];
         git(moduleDir, ["reset", "-q", "--hard", "HEAD"]);
         git(moduleDir, ["clean", "-q", "-fd"]);
+        record({ kind: "history.reset", undone: changed.length, paths: changed.slice(0, 20) }, placeOf(moduleDir));
         return changed;
     }
     catch {
@@ -235,6 +286,7 @@ export function abandonWork(start, moduleName, why) {
             git(moduleDir, ["reset", "-q", "--hard", commit]);
             git(moduleDir, ["clean", "-q", "-fd"]);
             cleanTestsBuild(moduleDir);
+            record({ kind: "history.rolledBack", why: firstLine(why, 300) }, placeOf(moduleDir));
             reportActivity(`${moduleName}: ${why} - ${basename(moduleDir)} is back where the work started`);
         }
         catch {

package/dist/regenerate/loop.js

@@ -5,14 +5,15 @@ import { STDLIB_ROOT } from "../plain/stdlib.js";
 import { ACCEPTANCE_TESTS, FUNCTIONAL_REQUIREMENTS, REQUIRED_FUNCTIONAL_SPECS, TEST_REQUIREMENTS } from "../plain/spec.js";
 import { reportActivity, reportStage } from "./activity.js";
 import { llmChunkAgents, MAX_LINES_PER_JOB, MAX_SPECS_PER_JOB, planChunks, stepsOf } from "./chunk-plan.js";
-import { commitNotes, recordHistory, resetToLastCommit } from "./lineage.js";
+import { commitNotes, landWork, recordHistory, resetToLastCommit } from "./lineage.js";
+import { firstLine, record } from "./ledger.js";
 import { NOTES_FILE, recordPulse } from "./notes.js";
 import { createJobRegistry, formatDuration } from "./jobs.js";
 import { moduleLanes } from "./lanes.js";
 import { adoptDigests, chainTree, codeTree, conformanceTarget, marksRegenerateAgain, REGENERATE_AGAIN, inheritedRecord, notBuilt, readRecord, RECORD_FILE, withResourceDigests, writeCodeRecord, writeTestsRecord, } from "./records.js";
 import { describeSpecDiffConformance, renderSpecDiffConformance } from "./conform.js";
 import { conformanceDirFor } from "./conformance.js";
-import { modulesRoot } from "./paths.js";
+import { modulesRoot, projectRootOf } from "./paths.js";
 import { budgetFor, choosingWhen, configForSpec, conformanceSwitchedOn, stackForModule, stackPinned, withStack } from "../config.js";
 import { describeSpecChange } from "./prompt.js";
 import { resolveTemplateDirs } from "./paths.js";
@@ -191,6 +192,11 @@ export function createWorkLoop(options) {
     // One tick at a time: each reads the records and the spec, and decides.
     let ticking = Promise.resolve();
     const tipEntry = (state) => tipOf(state.chain);
+    /** Where what the loop decides for a module is recorded: its tip's project, the module and the pulse it is about. */
+    const loopPlace = (state, job) => {
+        const tip = tipEntry(state);
+        return { project: projectRootOf(tip.buildDir), module: tip.moduleName, job };
+    };
     /** `work` with the codebase's stack chosen - the one it was asked for with - when it has one pinned. */
     const inStack = (state, work) => (state.stack !== undefined ? withStack(state.stack.name, work) : work());
     const keyOf = (spec) => serializeTree(codeTree(spec.tree));
@@ -681,6 +687,7 @@ export function createWorkLoop(options) {
                     /* a spec that does not parse halts it all the same */
                 }
                 state.halted = { why: `${why} - nothing more is built on it until its conformance tests pass`, specKey: key, pulse: pulseOf(running), ...(settled?.status === "stopped" ? {} : causeOf(settled)) };
+                record({ kind: "loop.halted", why: firstLine(state.halted.why, 300), track: "conformance" }, loopPlace(state, running.jobId));
             }
         }
         else if (running !== undefined && !landed) {
@@ -700,6 +707,7 @@ export function createWorkLoop(options) {
                 }
                 const why = settled?.status === "stopped" ? `pulse ${running.jobId} was stopped` : `pulse ${running.jobId} ("${running.planned.title}") did not land`;
                 state.halted = { why: `${why} - it runs again once the spec changes, or the module is asked for again`, specKey: key, pulse: pulseOf(running), ...(settled?.status === "stopped" ? {} : causeOf(settled)) };
+                record({ kind: "loop.halted", why: firstLine(state.halted.why, 300), track: "code" }, loopPlace(state, running.jobId));
             }
         }
         else {
@@ -866,6 +874,7 @@ export function createWorkLoop(options) {
                 else {
                     state.running.stopping = true;
                     state.why = `pulse ${state.running.jobId} was stopped: ${why}, which it was regenerating from`;
+                    record({ kind: "loop.stopped", why: firstLine(why, 300) }, loopPlace(state, state.running.jobId));
                     reportActivity(`  ${tip.moduleName}: ${state.why}`);
                     options.registry.stop(state.running.jobId, `Stopped by turri, not by you: ${why}, which it was regenerating from. What it had done is rolled back, and the spec as it is now is regenerated instead. To have a running pulse finish first and the change regenerated on top of it, set allow-to-stop-jobs: false in config.turri.yaml.`);
                     return;
@@ -1608,7 +1617,7 @@ const defaultConformanceRunner = async (job, onDelta) => {
     });
     if (result.passed) {
         writeTestsRecord(result.conformanceDir, job.planned.step.tree);
-        recordHistory(job.buildDir, `conformance tests: ${job.planned.title}`);
+        landWork(job.buildDir, `conformance tests: ${job.planned.title}`);
     }
     return { landed: result.passed, report: describeSpecDiffConformance(result), ...(result.specGaps !== undefined ? { specGaps: result.specGaps } : {}) };
 };

package/dist/regenerate/pulse-files.d.ts

@@ -0,0 +1,14 @@
+/** A file the running pulse wrote for a person to read - removed as it ends, unless in debug mode. Outside a pulse, nothing. */
+export declare function pulseWrote(file: string, job?: string | undefined): void;
+/**
+ * A pulse has ended. Unless the project is in debug mode, what it wrote for a person goes - its files, and its module's
+ * temporary folder - and so do the dependencies installed in the modules below its own in its chain. `lanes` are the
+ * pulse's (`plain_modules/<module>` of its whole chain), still held: no other pulse works in them. Returns whether
+ * everything was kept.
+ */
+export declare function pulseEnded(options: {
+    job: string;
+    project: string;
+    module?: string;
+    lanes: string[];
+}): boolean;

package/dist/regenerate/pulse-files.js

@@ -0,0 +1,63 @@
+import { existsSync, rmSync } from "node:fs";
+import { basename, join } from "node:path";
+import { debugMode } from "../config.js";
+import { currentJob } from "./activity.js";
+import { modulesScratchDir } from "./paths.js";
+/**
+ * What a pulse writes only for a person debugging turri, and when it goes.
+ *
+ * A pulse writes its own log, every one of its agents' whole logs, the testing scripts' and services' full outputs and
+ * its agents' temporary files. Nothing of turri reads them once the pulse is over: while it runs, its agents read their
+ * outputs (`read_output`, `service_logs`) and the diagnosis of a failure reads the logs - so they are written as before,
+ * and removed as it ends. In debug mode (`debug: true`, TURRI_DEBUG, `--debug`) they are kept, as they always were.
+ *
+ * So are the dependencies installed in the modules below a pulse's own in its chain: the chain is built in its tip, and
+ * what was installed in the codebases it left behind is only installed again - from the package cache - if one of them
+ * is ever built on again. The testing scripts install what they run with themselves.
+ */
+/** The files each pulse wrote for a person to read, by its id. */
+const written = new Map();
+/** A file the running pulse wrote for a person to read - removed as it ends, unless in debug mode. Outside a pulse, nothing. */
+export function pulseWrote(file, job = currentJob()?.id) {
+    if (job === undefined)
+        return;
+    const files = written.get(job) ?? new Set();
+    files.add(file);
+    written.set(job, files);
+}
+/** Where a codebase's dependencies are installed: what a module its chain has left behind keeps no longer. */
+const INSTALLED = ["node_modules", ".venv", "venv"];
+/**
+ * A pulse has ended. Unless the project is in debug mode, what it wrote for a person goes - its files, and its module's
+ * temporary folder - and so do the dependencies installed in the modules below its own in its chain. `lanes` are the
+ * pulse's (`plain_modules/<module>` of its whole chain), still held: no other pulse works in them. Returns whether
+ * everything was kept.
+ */
+export function pulseEnded(options) {
+    const files = written.get(options.job) ?? new Set();
+    written.delete(options.job);
+    if (debugMode(options.project))
+        return true;
+    const gone = (path) => {
+        try {
+            rmSync(path, { recursive: true, force: true });
+        }
+        catch {
+            /* what cannot be removed stays: never a reason for the pulse to fail */
+        }
+    };
+    for (const file of files)
+        gone(file);
+    for (const lane of options.lanes) {
+        // A codebase of the chain: its folder holds `code` - a lane may also be a spec file, or a build folder.
+        if (!existsSync(join(lane, "code")))
+            continue;
+        const name = basename(lane);
+        if (name === options.module)
+            gone(join(modulesScratchDir(lane), "tmp", name));
+        else
+            for (const installed of INSTALLED)
+                gone(join(lane, "code", installed));
+    }
+    return false;
+}

package/dist/regenerate/regenerate.js

@@ -11,7 +11,8 @@ import { debugFailure, diagnosisText } from "./debugger.js";
 import { codebaseMap, diffFiles, digestFiles, wroteNothing } from "./codebase.js";
 import { confinedTools } from "./confine.js";
 import { buildSpecDiffPrompt, describeSpecChange, reviewedSpec, sectionModules } from "./prompt.js";
-import { abandonWork, recordHistory, startWork } from "./lineage.js";
+import { abandonWork, landWork, startWork } from "./lineage.js";
+import { healing, record, signatureHash } from "./ledger.js";
 import { createWorkLedger, SUBMIT_WORK_NOTE } from "./submit-work.js";
 import { CODER_COMPACTION } from "./compaction.js";
 import { gapMark, gapsSince, NOTES_TURN, notesForSession, passedOutcome, recordNote, withNotes } from "./notes.js";
@@ -193,7 +194,7 @@ export async function renderSpecDiff(options) {
     if (builtBefore !== null && sameSpec(builtBefore, tree) && !dryRun) {
         stage("the specs were only restructured - nothing to render; recording the spec as it is now laid out");
         writeCodeRecord(tip.buildDir, tree);
-        recordHistory(tip.buildDir, `spec restructured: ${summarizeDiff(options.diff)}`, options.diff);
+        landWork(tip.buildDir, `spec restructured: ${summarizeDiff(options.diff)}`, options.diff);
         return {
             moduleName,
             buildDir: tip.buildDir,
@@ -440,7 +441,7 @@ export async function renderSpecDiff(options) {
                             : timedOut && seen < STUCK_ROUNDS
                                 ? `the unit testing script did not finish in ${Math.round(verdict.run.timeoutMs / 1000)}s - a debugger works out whether something hangs`
                                 : `the same failure ${seen} times - a debugger looks at it with fresh eyes`);
-                        const found = await debugFailure({
+                        const found = await healing("debugger", () => debugFailure({
                             buildDir,
                             failure: verdict.run.output,
                             times,
@@ -450,7 +451,7 @@ export async function renderSpecDiff(options) {
                             label: `${moduleName} spec change`,
                             ...(debuggerModel !== undefined ? { model: debuggerModel } : {}),
                             onDelta: options.onDelta,
-                        }).catch(() => undefined);
+                        }), (finding) => finding?.rootCause);
                         // Nothing hangs: the suite needs more time than a test run may take, and no agent can give it that.
                         if (found?.tooSlow === true) {
                             stage(`the unit testing script needs more than the ${Math.round(verdict.run.timeoutMs / 1000)}s a test run may take - stopping`);
@@ -477,6 +478,7 @@ export async function renderSpecDiff(options) {
                             ].join("\n");
                         }
                     }
+                    record({ kind: "gate.round", gate: "unit tests", round: followUp + 1, of: MAX_FOLLOW_UPS, ...(key !== undefined ? { signature: signatureHash(key) } : {}) });
                     stage(`${verdict.staticOnly ? "static checks" : "unit tests"} not passing - fixing (round ${followUp + 1} of ${MAX_FOLLOW_UPS})`);
                     response = await ask(executor, prompt, options.onDelta);
                     continue;
@@ -554,7 +556,7 @@ export async function renderSpecDiff(options) {
         stage("recording what the code is built from, and committing it");
         // Steered, it is built from the spec as it was steered to - the record the loop moved with it.
         writeCodeRecord(buildDir, steering?.target ?? tree);
-        recordHistory(buildDir, `spec diff: ${summarizeDiff(options.diff)}`, options.diff);
+        landWork(buildDir, `spec diff: ${summarizeDiff(options.diff)}`, options.diff);
     };
     if (!unitPassed) {
         stage(unitReviewStopped ? "the unit tests pass but still leave the change untested - undoing the change" : "unit tests NOT passing - undoing the change");

package/dist/regenerate/regression.js

@@ -6,7 +6,8 @@ import { reportActivity, reportStage, unitOf } from "./activity.js";
 import { confinedTools } from "./confine.js";
 import { projectRootOf } from "./paths.js";
 import { buildRegressionFixPrompt, sectionModules } from "./prompt.js";
-import { abandonWork, markWork, recordHistory } from "./lineage.js";
+import { abandonWork, landWork, markWork } from "./lineage.js";
+import { record } from "./ledger.js";
 import { missingToolchainError } from "./environment-error.js";
 import { workspaceBlock } from "./workspace.js";
 import { CODER_COMPACTION } from "./compaction.js";
@@ -154,13 +155,14 @@ export async function ensureRegressionPasses(input) {
         if (passing) {
             reportStage(`${unitOf(input.moduleName)}: the whole conformance suite passes, and the unit tests with it${repairs.length > 0 ? ` after ${repairs.length} repair${repairs.length === 1 ? "" : "s"}` : ""}`);
             if (repairs.length > 0) {
-                recordHistory(input.buildDir, `regression ${input.moduleName} (pass after ${repairs.length} repair(s))`);
+                landWork(input.buildDir, `regression ${input.moduleName} (pass after ${repairs.length} repair(s))`);
             }
             return { passed: true, runs, repairs, command };
         }
         if (round === maxRounds)
             break;
         input.onEvent?.({ type: "repair", frids: input.frids });
+        record({ kind: "gate.round", gate: "regression", round: round + 1, of: maxRounds });
         reportStage(`${unitOf(input.moduleName)}: an earlier functionality broke - repairing (round ${round + 1} of ${maxRounds})`);
         repairs.push(await repairRegression({
             testsDirs: dirs,

package/dist/regenerate/services.js

@@ -8,6 +8,7 @@ import { agentShell } from "./agent-shell.js";
 import { maskSecrets } from "./masking.js";
 import { modulesScratchDir } from "./paths.js";
 import { isRunning, startInBackground, stopGroup } from "./shell.js";
+import { pulseWrote } from "./pulse-files.js";
 /**
  * What an agent starts and keeps running while it works - a server, a worker - to try the software as someone would,
  * from outside: with curl, or what the stack's own client is. Whatever the stack: the agent gives the command that
@@ -83,6 +84,8 @@ export function createServiceTools(options) {
                 return say(`Not started: ${checked.reason}`);
             mkdirSync(logsDir, { recursive: true });
             const log = join(logsDir, `${name.replace(/[^\w.-]+/g, "_")}-${randomUUID().slice(0, 8)}.log`);
+            // Read through service_logs while the pulse runs, and by no one after it (pulse-files.ts).
+            pulseWrote(log);
             const limit = Math.min(Math.max(seconds ?? DEFAULT_READY_SECONDS, 1), MAX_READY_SECONDS);
             // Started and waited on under the audit: what it writes while it comes up, outside its role, is put back.
             const { value: outcome, note } = await fenced.audited(async () => {

package/dist/regenerate/side-panel.d.ts

@@ -17,6 +17,8 @@ import type { TranscriptStyle } from "./transcript-view.ts";
 export interface SidePanelOptions {
     /** Every job, oldest first. */
     jobs: () => Job[];
+    /** Earlier sessions' pulses, from the ledger: listed under this session's, in a section of their own. */
+    earlier?: () => Job[];
     /** The regeneration's pages. Not given: the pulse list alone. */
     map?: () => MarkedPage[];
     /** Rows the panel may use. */

package/dist/regenerate/side-panel.js

@@ -1,5 +1,5 @@
 import { Key, matchesKey, truncateToWidth, visibleWidth, wrapTextWithAnsi } from "@earendil-works/pi-tui";
-import { createJobsTreeState, jobsTreeLines, jobsTreeRows, keepCursorInView, scrollTree, treeKey } from "./jobs-tree.js";
+import { createJobsTreeState, jobsTreeLines, jobsTreeRows, keepCursorInView, scrollTree, sectionOf, treeKey } from "./jobs-tree.js";
 import { columnHeadings, createSpecMapState, DISCUSS_KEY, legendLines, minimap, shownPage, specMapHeader, specMapLines, switchTab, tabBar } from "./spec-map-view.js";
 export function createSidePanel(options) {
     const style = options.style;
@@ -71,7 +71,8 @@ export function createSidePanel(options) {
         const bodyHeight = Math.max(3, rows - top.length - 2);
         lastBodyHeight = bodyHeight;
         // The tree scrolls to keep the cursor in sight, and only that far.
-        const tree_ = jobsTreeLines(options.jobs(), tree, width, style);
+        // This session's pulses, and under them the earlier sessions' the ledger has.
+        const tree_ = jobsTreeLines([...options.jobs(), ...(options.earlier?.() ?? [])], tree, width, style);
         drawnTree = { rows: tree_.rows, rowStarts: tree_.rowStarts, lineCount: tree_.lines.length };
         keepCursorInView(tree, tree_.rowStarts, tree_.lines.length, bodyHeight);
         const visible = tree_.lines.slice(tree.top, tree.top + bodyHeight);
@@ -132,7 +133,7 @@ export function createSidePanel(options) {
         const job = jobs.find((entry) => entry.id === id);
         if (job === undefined)
             return;
-        const section = job.status === "running" ? "running" : job.status === "queued" ? "queued" : "finished";
+        const section = sectionOf(job);
         tree.open[section] = true;
         const at = jobsTreeRows(jobs, tree).findIndex((entry) => entry.kind === "job" && entry.job.id === id);
         if (at !== -1)

package/dist/regenerate/test-scripts.js

@@ -4,6 +4,7 @@ import { accessSync, constants, copyFileSync, existsSync, mkdirSync, readdirSync
 import { basename, delimiter, dirname, join, resolve, sep } from "node:path";
 import { CONFIG_FILE, CONFIG_KEYS, projectConfigFile, readProjectConfig, stackConfigFile, stackConfigFileIn, stackOf, stacksOf } from "../config.js";
 import { reportActivity, reportLogNote, reportNote } from "./activity.js";
+import { record, signatureHash } from "./ledger.js";
 import { ensureInModulesScratch, modulesScratchDir } from "./paths.js";
 import { defineTool } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
@@ -361,7 +362,7 @@ async function runScript(cwd, script, args, logDir, options = {}) {
     const noted = withScriptLineNote(text, script);
     // Its failures, counts and end are what the agent is shown; the whole is kept, a read_output away.
     const log = keepTestOutput(logDir, basename(script), noted);
-    return {
+    const run = {
         command: shown(script),
         exitCode: timedOut === true ? 124 : exitCode,
         output: shownTestOutput(noted, log),
@@ -370,6 +371,18 @@ async function runScript(cwd, script, args, logDir, options = {}) {
         elapsedMs: Date.now() - started,
         ...(log !== undefined ? { log } : {}),
     };
+    record({
+        kind: "script.run",
+        script: basename(script),
+        exit: run.exitCode,
+        ms: run.elapsedMs,
+        timedOut: run.timedOut === true,
+        inPlace: options.named === undefined,
+        // On code changed for the run - can_it_fail's bug: failing is what it is for.
+        ...(options.mutated === true ? { mutated: true } : {}),
+        ...(run.exitCode !== 0 ? { signature: signatureHash(failureSignature(run)) } : {}),
+    });
+    return run;
 }
 /** What bash says when a line of the script itself cannot run: a command or file that is not there, or not runnable. */
 const OWN_LINE_FAILURE = /: (No such file or directory|command not found|Permission denied)$/;
@@ -406,10 +419,10 @@ function quote(path) {
     return `'${path.replace(/'/g, "'\\''")}'`;
 }
 /** A testing script run on a fresh copy of the code: the project's script copied fresh into the staging folder, beside it. */
-async function runOnCopy(stagingRoot, script, args, logDir, limitMs) {
+async function runOnCopy(stagingRoot, script, args, logDir, limitMs, mutated) {
     const copy = join(stagingRoot, basename(script));
     writeFileSync(copy, readFileSync(script));
-    return runScript(stagingRoot, script, args, logDir, { named: copy, ...(limitMs !== undefined ? { limitMs } : {}) });
+    return runScript(stagingRoot, script, args, logDir, { named: copy, ...(limitMs !== undefined ? { limitMs } : {}), ...(mutated === true ? { mutated } : {}) });
 }
 /**
  * The harness's run of the unit testing script on the code as it is now - and run_unit_tests'.
@@ -475,7 +488,7 @@ limitMs) {
         return run;
     };
     if (project.inPlace) {
-        const inPlace = () => runScript(project.root, script, [resolve(buildDir)], buildDir, limitMs !== undefined ? { limitMs } : {});
+        const inPlace = () => runScript(project.root, script, [resolve(buildDir)], buildDir, { ...(limitMs !== undefined ? { limitMs } : {}), ...(change !== undefined ? { mutated: true } : {}) });
         const started = Date.now();
         if (change === undefined)
             return passed(await inPlace(), started);
@@ -490,7 +503,7 @@ limitMs) {
         if (change !== undefined)
             writeFileSync(join(staging.build, change.file), change.text);
         const started = Date.now();
-        return passed(await runOnCopy(staging.root, script, [staging.build], buildDir, limitMs), started);
+        return passed(await runOnCopy(staging.root, script, [staging.build], buildDir, limitMs, change !== undefined), started);
     }
     finally {
         rmSync(staging.root, { recursive: true, force: true });

package/dist/regenerate/testing.js

@@ -9,6 +9,7 @@ import { modulesScratchDir } from "./paths.js";
 import { RECORD_FILE } from "./records.js";
 import { missingToolchain, runUnitScript, scriptFailure } from "./test-scripts.js";
 import { COMMAND_TIMEOUT_MS } from "./shell.js";
+import { pulseWrote } from "./pulse-files.js";
 /** Up to this much of a run's output is shown: all of it when it fits, otherwise its end. */
 const MAX_OUTPUT_CHARS = 60_000;
 /** A test run has the same time limit as any command: see COMMAND_TIMEOUT_MS. */
@@ -92,6 +93,8 @@ export function keepOutput(dir, label, output) {
         const name = label.replace(/[^\w.-]+/g, "_").slice(0, 60);
         const file = join(logs, `${stamp}-${name}.log`);
         writeFileSync(file, typeof output === "string" ? maskSecrets(output) : maskSecrets(output.toString("utf8")));
+        // Read through read_output while the pulse runs, and by no one after it (pulse-files.ts).
+        pulseWrote(file);
         const all = readdirSync(logs).sort();
         for (const old of all.slice(0, Math.max(0, all.length - MAX_LOGS)))
             rmSync(join(logs, old), { force: true });

package/dist/regenerate/unit-review.js

@@ -3,6 +3,7 @@ import { defineTool } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
 import { ask, createExecutor } from "../pi.js";
 import { reportActivity } from "./activity.js";
+import { record } from "./ledger.js";
 import { diffFiles, digestFiles } from "./codebase.js";
 import { confinedTools } from "./confine.js";
 import { REVIEWER_CALLS, reviewKind } from "./conformance-agents.js";
@@ -288,6 +289,7 @@ async function reviewLoop(options, reviewer) {
         letGo = [];
         options.stage(`reviewing the code and its unit tests (review ${reviews + 1}): does the code do what the spec says, and does each test fail when that breaks`);
         const review = await reviewer.review(request);
+        record({ kind: "review.verdict", reviewer: "unit", review: reviews + 1, pass: review.pass, problems: review.pass ? 0 : review.problems.length });
         if (review.pass) {
             options.stage("the review has nothing left to improve in the code or its unit tests");
             return waived.length > 0 ? { passed: true, waived, reviews: reviews + 1 } : { passed: true, reviews: reviews + 1 };
@@ -316,6 +318,7 @@ async function reviewLoop(options, reviewer) {
             });
             // No ruling: the review stands, as it did before there was an arbiter.
             if (ruling !== undefined) {
+                record({ kind: "arbiter.ruling", upheld: ruling.upheld.length, dismissed: ruling.dismissed.length });
                 waived.push(...ruling.dismissed.map((entry) => entry.problem));
                 if (ruling.dismissed.length > 0)
                     recordNote(options.buildDir, "pitfalls", dismissedNote(ruling, "unit test"), `${options.label} unit tests`);

package/dist/run-report.js

@@ -1,5 +1,6 @@
 import { mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { basename, dirname, join, relative, resolve } from "node:path";
+import { debugMode } from "./config.js";
 import { turriIdentity } from "./environment.js";
 import { onModelEvent } from "./regenerate/activity.js";
 import { firstRegeneration } from "./regenerate/jobs.js";
@@ -290,11 +291,14 @@ export function startRunReport(options) {
             spec_gaps: specGapsOf(jobs),
             turn_incidents: turnIncidentsOf(),
             ...(exitParts ?? {}),
-            logs: {
-                jobs_dir: fromProject(join(projectRoot, ".turri", "jobs")),
-                debug_dir: fromProject(join(projectRoot, ".turri", "debug")),
-                sessions_dir: fromProject(join(projectRoot, ".turri", "sessions")),
-            },
+            // Kept only in debug mode: otherwise each pulse's go as it ends, and the sessions are never written.
+            logs: debugMode(projectRoot)
+                ? {
+                    jobs_dir: fromProject(join(projectRoot, ".turri", "jobs")),
+                    debug_dir: fromProject(join(projectRoot, ".turri", "debug")),
+                    sessions_dir: fromProject(join(projectRoot, ".turri", "sessions")),
+                }
+                : null,
         };
     };
     const write = (exitCode) => {

package/docs/config.md

@@ -42,11 +42,12 @@ Each change says when it takes effect. None needs turri started again - what tur
 | --- | --- |
 | now | `harness-model` and `models.chat` - the conversation switches to it; `parallel-jobs` - a pulse waiting for a slot starts if there is one now; `allow-to-stop-jobs` |
 | your next message | `rules-off` - for the conversation; a skill's agents from the next skill run |
-| the next pulse - one running keeps what it started with | `coding-model`, `judge-model`, every other `models.<role>`, `review`, `mutation-testing`, `turn-limit`, `compact-at`, `agent-env`, `toolchains` |
+| the next pulse - one running keeps what it started with | `coding-model`, `judge-model`, every other `models.<role>`, `review`, `mutation-testing`, `turn-limit`, `compact-at`, `agent-env`, `toolchains`, `debug` |
 | the next time the work is planned | `conformance-tests`, `conformance-budget`, `max-specs-per-pulse`, `max-lines-per-pulse` |
 | the next test run - the running pulse's too | the testing scripts, `testing-scripts-run-on` |
 | the next pulse that lands - the running one too | `release-after-pulse` |
 | the next regeneration that names no `.plain` file | `module` |
+| the next time turri starts here | `ledger-keep`, `cache-keep` |
 
 A coding model chosen this session - `--coding-model`, or `engine` - gives way to one set here: the last word wins. It says, too, what wins over the change: `models.chat` over `harness-model` (and so on), and a stack's `config.turri.<stack>.yaml`, which is not changed here, while that stack is chosen. In a project still read from `config.yaml`, the first change copies it to `config.turri.yaml` first - see [the old config.yaml](#the-old-configyaml).
 
@@ -91,6 +92,9 @@ models:                              # each agent role's model, thinking and tur
 | `conformance-tests-script` | the conformance testing script, by path from the project root | looked for - see below |
 | `prepare-environment-script` | optional: run as `<script> <build>` before every run of the conformance testing script, which runs only once it passes - see [the testing scripts](regenerating.md#the-testing-scripts). turri never writes one | the one `prepare_environment*.sh` at the root or in `test_scripts/`, else none |
 | `compact-at` | how many tokens of work - past its instructions - an agent kept through a job holds before the harness compacts it between turns; at least 1000; the project's alone | 100,000 - see [agent sessions](regenerating.md#agent-sessions) |
+| `ledger-keep` | how many megabytes of the ledger - what turri did, as facts, in `.turri/ledger` - the project keeps: past it, the oldest runs' are pruned when a run first writes; `0` keeps none, and writes none; the project's alone. See [the ledger](using.md#the-ledger) | 100 |
+| `debug` | `true` or `false`: whether turri keeps what it writes only for a person debugging it - every agent's whole log and saved session, each pulse's log, the whole outputs of its test runs and services, its agents' temporary files, and the dependencies installed in the modules a chain has left behind. `false`: only what turri reads to work is kept. `TURRI_DEBUG=1` - or `turri --debug` - turns it on for one run, and `TURRI_DEBUG=0` off; the project's alone. See [debug mode](#debug-mode) | `false` |
+| `cache-keep` | how many megabytes of package caches the project keeps - `plain_modules/.turri/cache`, and the caches in the agents' home folders: past it, the least recently written - by the newest file in each - are removed when a run starts its first agent command, each downloaded again when it is wanted; `0` keeps none; the project's alone | 4096 |
 | `max-specs-per-pulse` | the most functional specs the planner groups into one pulse - what builds one feature goes together, up to this many; a stack's file wins | 3 |
 | `max-lines-per-pulse` | the most lines of spec in one pulse - its functional specs with the definitions they bring; a group past it is split again, a functional spec a pulse; a stack's file wins | 120 |
 | `parallel-jobs` | how many jobs the chat runs at once - modules that share nothing build side by side up to it; read when the chat starts, from the folder it starts in | half this machine's cores, between one and four - see [how many jobs at once](#how-many-jobs-at-once) |
@@ -178,6 +182,23 @@ The optional prepare script - `prepare-environment-script:`, else the one `prepa
 
 A folder holding two of a kind is not guessed between: name the one to use. A script named inside `plain_modules/` is refused - agents write there, so it could be changed by the work it judges. The scripts are never linked from the specs. What they are and how they run is in [regenerating the code](regenerating.md#the-testing-scripts).
 
+## Debug mode
+
+turri writes much more than it needs to work, for a person working out why it did what it did - each agent's whole conversation, each pulse's whole transcript, every test run's whole output. Most of it nothing of turri ever reads again. So by default it is not kept:
+
+| | outside debug mode | in debug mode |
+| --- | --- | --- |
+| each pulse's log (`.turri/jobs/`) and its agents' (`.turri/debug/`) | written while the pulse runs - the diagnosis of a failure reads them - and removed as it ends | kept |
+| the logs of agents outside a pulse - the release agent, a side question | not written | kept |
+| the agents' saved sessions (`.turri/sessions/`) | not written - each is kept in memory while it works; a gated skill's alone is saved, as its run resumes from it | kept |
+| whole outputs of test runs and services (`plain_modules/.turri/logs/`, `services/`) | read by the pulse's agents while it runs, removed as it ends | kept, the last 100 outputs |
+| a module's `TMPDIR` (`plain_modules/.turri/tmp/<module>`) | emptied as each of its pulses ends | kept |
+| dependencies installed in the modules below a pulse's own in its chain (`node_modules`, `.venv`) | removed as the pulse ends: the chain is built in its tip, and a module built on again installs them again, from the cache | kept |
+
+Kept either way: the ledger - what each pulse did, how it ended, what it cost (see [the ledger](using.md#the-ledger)) - the code, its tests and their history, a gated skill's run, what a conformance explorer found, what `deploy` started, and the package caches, held to `cache-keep:`. The testing scripts install what they run with themselves, so nothing removed is ever missed by a test run.
+
+Turn it on in `config.turri.yaml` (`debug: true`, or `/turri-config debug true`), or for one run with `TURRI_DEBUG=1` or `turri --debug`.
+
 ## Credentials for the agents
 
 An agent's commands do not get the environment turri was started in: it holds your providers' API keys, cloud credentials and tokens. They get only the path, the locale, the terminal and the user, the variables that locate your toolchains (`JAVA_HOME`, `NVM_DIR`, `RUSTUP_HOME`, ... - the ones in your home found even when your shell sets none), and turri's own: a home folder of the module's own (`plain_modules/.turri/home/<module>`, kept between pulses, with a git identity of its own), a `TMPDIR`, the package caches in `plain_modules/.turri/cache`, and git stopped at the module folder, so a project that is itself a repository is never found from the code.
@@ -229,7 +250,7 @@ How a stack's config is made:
 - **Laid over `config.turri.yaml`, key by key.** What the stack's file gives wins; what it leaves out is the project's. A model it names takes what it leaves out from the project's same model - above, with the stack frontend chosen, everything renders on the project's coding model, thinking `high`.
 - **Pinned to a module when it is asked for.** A module is built with the stack it was asked for with, from its first pulse to its last: the pulses planned after it, its test runs, the scripts found, its plans and its conformance tests, whatever stack is chosen meanwhile. A pulse waiting for a free slot keeps the stack it was queued with. A change to its spec keeps its stack - unless it was built with none and has a stack of its own now; asked for again, its stack is worked out again. What `regenerate` says back names each module's stack, and so does a pulse's report.
 - **Refused with none, when that cannot build.** In a project of stacks, a module asked for with no stack - none named, none named after it, none chosen - is refused before anything is planned when `config.turri.yaml` alone names no unit testing script: every pulse would stop for want of one. The refusal names the stacks and the script each names. A spec change `write_spec_diff` writes is never refused: it is written, and said not to be built yet - passed the stack, everything written is built. `stack: none` builds with `config.turri.yaml` alone all the same.
-- **The project's alone.** `module:`, `harness-model:`, `parallel-jobs:`, `rules-off:` and `compact-at:` are read only from `config.turri.yaml` - there is one conversation, one module built when none is named, one machine. `agent-env:` and `toolchains:` in a stack's file add to the project's.
+- **The project's alone.** `module:`, `harness-model:`, `parallel-jobs:`, `rules-off:`, `compact-at:`, `ledger-keep:`, `debug:` and `cache-keep:` are read only from `config.turri.yaml` - there is one conversation, one module built when none is named, one machine. `agent-env:` and `toolchains:` in a stack's file add to the project's.
 
 The chat is told each stack's config and the one chosen. The skills that write testing scripts put each stack's in its own file once a project has two stacks, each stack named after the module it builds.
 

package/docs/development.md

@@ -56,7 +56,9 @@ The workflow signs in to npm with npm's trusted publishing (OIDC), not a token:
 | `src/regenerate/side-panel.ts`, `src/regenerate/panel-dock.ts` | the side panel, and the chat narrowed beside it when it is docked |
 | `src/screen-padding.ts` | the blank row above and below the whole screen and the two columns either side, in both of pi's ways of drawing it |
 | `src/regenerate/spec-map.ts`, `src/regenerate/spec-map-view.ts` | regen: each item's mark, from the records, the `.plain` files and the work loop's plan (`mapPlan`), and the map drawn |
-| `src/regenerate/debug-log.ts` | each render agent's whole conversation, always streamed to `.turri/debug/` |
+| `src/regenerate/debug-log.ts` | each render agent's whole conversation, streamed to `.turri/debug/` - while its pulse runs, and kept in debug mode |
+| `src/regenerate/pulse-files.ts` | what a pulse writes only for a person - its logs, outputs, temporary files, its chain's installed dependencies - removed as it ends outside debug mode |
+| `src/regenerate/ledger.ts`, `src/regenerate/ledger-view.ts`, `src/ledger-tools.ts`, `src/ledger-cli.ts` | the ledger: what turri did, as typed facts, in `.turri/ledger/` - `record()`, read back for the `ledger` tool, `pulses`, `diagnose` and `turri ledger` - see [the ledger](using.md#the-ledger) |
 | `genome/` | what turri knows and can do: `genome/skills/`, the skill library, and `genome/rules/`, the \*\*\*plain authoring rules |
 
 ## Known edges