backpass 0.1.7 → 0.1.8

package/src/apply/writer.js

@@ -1,11 +1,19 @@
 import fs from "node:fs";
 import path from "node:path";
+import { randomUUID } from "node:crypto";
 
 import { applyEdit, projectWithDecisions } from "../proposal.js";
 import { memoryTextHash } from "../memory.js";
 import { budgetGateKind, budgetStatus, formatTokens } from "../tokens.js";
 import { recordRejection } from "../state.js";
-import { writeSkill } from "../skills.js";
+import {
+  CANONICAL_SKILLS_DIR,
+  CLAUDE_SKILLS_LINK,
+  editSkills,
+  ensureSkillsLayout,
+  removeOwnedSkillPaths,
+  writeSkill,
+} from "../skills.js";
 
 function acceptedSubsetBudgetFailure({ proposal, accepted, repo, capTokens, memoryText }) {
   if (!accepted.length) return null;
@@ -98,16 +106,77 @@ function overBudgetWarning(relative, budget) {
   );
 }
 
+function atomicReplace(target, text) {
+  const temp = path.join(path.dirname(target), `.${path.basename(target)}.backpass-${randomUUID()}`);
+  const mode = fs.statSync(target).mode & 0o7777;
+  let fd;
+  let ownership;
+  try {
+    fd = fs.openSync(temp, "wx");
+    const stat = fs.fstatSync(fd);
+    ownership = [{ absolute: temp, identity: { dev: stat.dev, ino: stat.ino } }];
+    fs.fchmodSync(fd, mode);
+    fs.writeFileSync(fd, text);
+    fs.fsyncSync(fd);
+    fs.closeSync(fd);
+    fd = undefined;
+    fs.renameSync(temp, target);
+    return { absolute: target, identity: ownership[0].identity, text };
+  } catch (err) {
+    if (fd !== undefined) {
+      try {
+        fs.closeSync(fd);
+      } catch {
+        // Preserve the original write error.
+      }
+    }
+    removeOwnedSkillPaths(ownership ?? []);
+    throw err;
+  }
+}
+
+function commitStillCurrent(commit) {
+  let stat;
+  try {
+    stat = fs.lstatSync(commit.absolute);
+    if (stat.dev !== commit.identity.dev || stat.ino !== commit.identity.ino) return false;
+    return fs.readFileSync(commit.absolute, "utf8") === commit.text;
+  } catch {
+    return false;
+  }
+}
+
+function absentParentDirectories(root, files) {
+  const directories = new Set();
+  for (const file of files) {
+    for (let current = path.dirname(file); current !== root; current = path.dirname(current)) {
+      if (!fs.existsSync(current)) directories.add(current);
+    }
+  }
+  return [...directories].sort((a, b) => b.length - a.length);
+}
+
+function removeEmptyDirectories(directories) {
+  for (const directory of directories) {
+    try {
+      fs.rmdirSync(directory);
+    } catch {
+      continue;
+    }
+  }
+}
+
 /**
  * The only place in backpass that writes to the repo.
  *
  * Everything upstream is read-only analysis; a run only changes the weights here, after
- * a human accepted specific edits. Three gates run before the first byte is written:
+ * a human accepted specific edits. Five gates run before the first byte is written:
  * the memory file must still be the file the proposal was measured against
  * (`memoryFileSnapshot`), the accepted subset must clear the same cap/shrink budget gate as
- * the full proposal (`budgetGateKind`), and every accepted edit for a file must compose
- * against that file's single pre-write image. Any of them failing writes nothing and
- * records no rejection.
+ * the full proposal (`budgetGateKind`), every accepted edit for a file must compose
+ * against that file's single pre-write image, every created skill target must still be
+ * absent, and accepted paths must resolve to distinct targets. Any of them failing writes
+ * nothing and records no rejection.
  *
  * A file is therefore applied all at once or not at all. Skills are written only after
  * every accepted edit has composed, and before the files that reference them.
@@ -197,33 +266,98 @@ export function applyDecisions({ proposal, decisions, repo, state, config, dryRu
     for (const id of applied) landed.add(id);
   }
 
+  const plannedTargets = new Map();
+  const resolvedPlanned = [];
+  for (const item of planned) {
+    let resolved;
+    try {
+      resolved = fs.realpathSync(item.absolute);
+    } catch (err) {
+      results.failed.push({ file: item.relative, error: `${item.relative} could not be resolved: ${err.message}` });
+      continue;
+    }
+    const existing = plannedTargets.get(resolved);
+    if (existing) {
+      results.failed.push({
+        file: item.relative,
+        error: `${item.relative} resolves to the same target as ${existing.relative}; nothing was written`,
+      });
+      continue;
+    }
+    const resolvedItem = { ...item, resolved };
+    plannedTargets.set(resolved, resolvedItem);
+    resolvedPlanned.push(resolvedItem);
+  }
+
   if (results.failed.length) return results;
 
   // Skills go in before the memory file. A skill nothing points at yet is inert, while a
   // memory file pointing at a skill that is not there is actively wrong - so if a skill
   // cannot be written, the files that would reference it are left alone.
-  const skillFailures = [];
-  const writtenSkillPaths = [];
+  const plannedSkills = [];
+  const skillPaths = new Set();
   for (const edit of accepted) {
-    if (edit.kind !== "extract" || !edit.skill) continue;
-    if (!landed.has(edit.id)) continue;
+    if (edit.kind !== "extract" || !landed.has(edit.id)) continue;
+    for (const skill of editSkills(edit)) {
+      const absolute = path.join(repo.root, skill.path);
+      if (skillPaths.has(skill.path) || fs.existsSync(absolute)) {
+        results.failed.push({
+          file: skill.path,
+          edit: edit.id,
+          error: `${skill.path} already exists; nothing was written`,
+        });
+      }
+      skillPaths.add(skill.path);
+      plannedSkills.push({ edit, skill });
+    }
+  }
+  if (results.failed.length) return results;
+
+  const canonical = plannedSkills.find(
+    ({ skill }) => skill.path === CANONICAL_SKILLS_DIR || skill.path.startsWith(`${CANONICAL_SKILLS_DIR}/`),
+  );
+  const createdDirectoryCandidates = absentParentDirectories(repo.root, [
+    ...plannedSkills.map(({ skill }) => path.join(repo.root, skill.path)),
+    ...(canonical ? [path.join(repo.root, CLAUDE_SKILLS_LINK)] : []),
+  ]);
+  const skillFailures = [];
+  const ownedSkillPaths = [];
+  for (const { edit, skill } of plannedSkills) {
     try {
-      const layout = dryRun ? { created: [], warnings: [] } : writeSkill(repo.root, edit.skill);
-      results.skills.push({ path: edit.skill.path, dryRun, created: layout.created });
-      if (!dryRun) writtenSkillPaths.push(edit.skill.path);
+      const layout = dryRun
+        ? { created: [], warnings: [] }
+        : writeSkill(repo.root, skill, { exclusive: true, ensureLayout: false });
+      results.skills.push({ path: skill.path, dryRun, created: layout.created });
+      if (!dryRun) {
+        ownedSkillPaths.push(...("ownership" in layout && Array.isArray(layout.ownership) ? layout.ownership : []));
+      }
       for (const w of layout.warnings) if (!results.warnings.includes(w)) results.warnings.push(w);
     } catch (err) {
-      skillFailures.push({ file: edit.skill.path, edit: edit.id, error: err.message });
+      skillFailures.push({ file: skill.path, edit: edit.id, error: err.message });
     }
   }
 
-  if (skillFailures.length) {
-    results.failed.push(...skillFailures);
-    if (writtenSkillPaths.length) {
+  const rollbackSkills = () => {
+    const { removed, conflicts } = removeOwnedSkillPaths(ownedSkillPaths);
+    removeEmptyDirectories(createdDirectoryCandidates);
+    results.skills = [];
+    const removedPaths = removed.map((item) => item.relative).filter(Boolean);
+    if (removedPaths.length) {
       results.failed.push({
-        error: `skill paths already written in this round: ${writtenSkillPaths.join(", ")}; remove them before retrying`,
+        error: `rolled back skill paths written earlier in this round: ${removedPaths.join(", ")}`,
       });
     }
+    for (const item of conflicts) {
+      results.failed.push({
+        file: item.relative,
+        error: `${item.relative} rollback conflict: the skill changed after this apply wrote it; left untouched`,
+      });
+    }
+  };
+
+  if (skillFailures.length) {
+    results.failed.push(...skillFailures);
+    rollbackSkills();
     for (const { relative } of planned) {
       results.failed.push({
         file: relative,
@@ -233,16 +367,69 @@ export function applyDecisions({ proposal, decisions, repo, state, config, dryRu
     return results;
   }
 
-  for (const { relative, absolute, before, text, applied } of planned) {
+  const orderedPlanned = [...resolvedPlanned].sort((a, b) => {
+    const aMemory = a.relative === proposal.memoryFile.path;
+    const bMemory = b.relative === proposal.memoryFile.path;
+    return Number(aMemory) - Number(bMemory);
+  });
+  const committed = [];
+  const rollbackCommitted = () => {
+    for (const written of [...committed].reverse()) {
+      if (!commitStillCurrent(written.commit)) {
+        results.failed.push({
+          file: written.relative,
+          error: `${written.relative} rollback conflict: the file changed after this apply wrote it; left untouched`,
+        });
+        continue;
+      }
+      try {
+        atomicReplace(written.commit.absolute, written.before);
+      } catch (rollbackError) {
+        results.failed.push({
+          file: written.relative,
+          error: `${written.relative} could not be rolled back: ${rollbackError.message}`,
+        });
+      }
+    }
+    results.written = [];
+  };
+  for (const item of orderedPlanned) {
+    const { relative, resolved, before, text, applied } = item;
     const budget = relative === proposal.memoryFile.path ? budgetStatus(before, text, config.budgetTokens) : null;
 
-    if (!dryRun) fs.writeFileSync(absolute, text);
+    let commit = null;
+    try {
+      if (!dryRun) commit = atomicReplace(resolved, text);
+    } catch (err) {
+      results.failed.push({
+        file: relative,
+        error: `${relative} could not be written: ${err.message}`,
+      });
+      rollbackCommitted();
+      rollbackSkills();
+      return results;
+    }
+    committed.push({ ...item, commit });
     results.written.push({ file: relative, edits: applied, budget, dryRun });
 
     // Shrinking over several runs is the design, so this is a heading, not a failure.
     if (budget && !budget.withinBudget) results.warnings.push(overBudgetWarning(relative, budget));
   }
 
+  if (!dryRun && canonical) {
+    try {
+      const layout = ensureSkillsLayout(repo.root);
+      const result = results.skills.find(({ path: skillPath }) => skillPath === canonical.skill.path);
+      result.created = [...new Set([...result.created, ...layout.created])];
+      for (const w of layout.warnings) if (!results.warnings.includes(w)) results.warnings.push(w);
+    } catch (err) {
+      results.failed.push({ file: CLAUDE_SKILLS_LINK, edit: canonical.edit.id, error: err.message });
+      rollbackCommitted();
+      rollbackSkills();
+      return results;
+    }
+  }
+
   // Rejections are remembered so the same edit is not re-proposed without new evidence.
   if (!dryRun && rejected.length) {
     const rejections = state.readRejections();

package/src/cli.js

@@ -74,7 +74,7 @@ COMMANDS
              gradient descent. Never writes.
   scan       collect samples only: which transcripts belong to this repo, and how we know
   analyze    calculate loss: one cheap model call per new transcript (tier 1)
-  propose    aggregate gradients, then gradient descent: one high-reasoning call
+  propose    aggregate gradients, then high-reasoning gradient descent
              turning the aggregated evidence into edits (tier 2)
   apply      review the proposal and write the accepted edits (the only writer)
   status     cache state, evidence counts, and the budget bar
@@ -99,10 +99,10 @@ MODELS (two-tier: cheap analysis, smart synthesis - all through acpx)
   Setting an agent pins that pass and skips its ladder.
   --analysis-agent <a>     acpx agent for the per-transcript pass       [auto]
   --analysis-model <id>    model id for the analysis pass (needs --analysis-agent)
-  --analysis-effort <e>    reasoning effort, when the adapter advertises it  [medium]
+  --analysis-effort <e>    one-off reasoning effort, when supported          [medium]
   --synthesis-agent <a>    acpx agent for the final proposal pass       [auto]
   --synthesis-model <id>   model id for the synthesis pass (needs --synthesis-agent)
-  --synthesis-effort <e>   reasoning effort for synthesis               [high]
+  --synthesis-effort <e>   one-off reasoning effort for synthesis            [high]
   --no-auto-agent          skip the ladders and pin codex / claude (the pre-0.2 defaults)
   --jobs <n>               parallel analysis calls                      [4]
 

package/src/commands/propose.js

@@ -35,6 +35,10 @@ export async function foldForRun(ctx, memoryFile) {
 
 export async function runProposal(ctx, precomputed = null) {
   const { repo, config } = ctx;
+  // Starting a new proposal run invalidates the previous result immediately. Discovery,
+  // folding, and agent resolution can all fail before synthesis starts; none of those
+  // failures may leave an older proposal available to apply as if it came from this run.
+  config.state.clearProposal();
   const { file } = precomputed || primaryMemoryFile(repo, config);
   const transcripts = precomputed?.transcripts || (await discoverForRun(ctx)).transcripts;
 
@@ -113,6 +117,60 @@ export function printProposal(proposal, { applied = false, analysisUsage = [] }
   out("Review and apply with `backpass apply` (nothing has been written).");
 }
 
+/** True when a violation is about the always-loaded budget rather than the annotation. */
+const isBudgetViolation = (v) => /-token budget/.test(v);
+const isEditCapViolation = (v) => /per-run cap is \d+/.test(v);
+
+/**
+ * What to actually try next, read off the condition the run ended on.
+ *
+ * The old advice - a stronger model, a bigger budget, a higher edit cap - was printed for
+ * every failure, including the ones where the model never spoke and the ones where the
+ * budget was never the constraint. Each terminal condition has a different repair.
+ */
+export function synthesisFailureHint(err) {
+  if (err.reason === "empty") {
+    return "the synthesis harness returned no text, so nothing about the model, the budget, or the edit cap was the constraint; run `backpass propose` again to start a fresh synthesis session";
+  }
+  if (err.reason === "unparseable") {
+    return "the model answered but not with a JSON object; run `backpass propose` again, or pin a different harness with --synthesis-agent";
+  }
+  if (err.reason === "editing") {
+    return "the agent kept rewriting the staging copy instead of describing it; run `backpass propose` again to start fresh";
+  }
+  const violations = err.violations || [];
+  if (violations.some(isBudgetViolation)) {
+    return "the edit set did not clear the budget gate: raise --budget, or let the shrink continue over more runs";
+  }
+  if (violations.some(isEditCapViolation)) {
+    return "the annotation proposed more edits than the per-run learning rate allows: raise --max-edits, or re-run and let the next pass take the rest";
+  }
+  return "the gates above are what the next synthesis must satisfy; run `backpass propose` again";
+}
+
+/**
+ * Report a synthesis that ended without a valid proposal: loudly, and about the turn that
+ * actually ended it (design section 6).
+ *
+ * The saved proposal and the terminal condition can be from different turns - a run whose
+ * last turn was empty leaves the rejected proposal of an earlier one on disk - so the
+ * provenance is printed rather than letting the older violations read as this turn's.
+ */
+export function printSynthesisFailure(err, state) {
+  info("");
+  for (const violation of err.violations) info(`  ${color.red("x")} ${violation}`);
+  info("");
+  if (!err.saved) {
+    info(color.dim("  no proposal was saved: no annotation turn produced one"));
+    return;
+  }
+  info(color.dim(`  the rejected proposal was saved to ${state.proposalPath}`));
+  if (err.reason !== "gates") {
+    info(color.dim(`  it is from annotation attempt ${err.saved.attempt}, not the turn above, and it lists:`));
+    for (const violation of err.saved.violations) info(color.dim(`    - ${violation}`));
+  }
+}
+
 export async function cmdPropose(ctx) {
   try {
     const { proposal } = await runProposal(ctx);
@@ -124,12 +182,8 @@ export async function cmdPropose(ctx) {
     return 0;
   } catch (err) {
     if (err instanceof ProposalViolation) {
-      // Loud failure, never silent truncation (design section 6).
-      info("");
-      for (const violation of err.violations) info(`  ${color.red("x")} ${violation}`);
-      info("");
-      info(color.dim(`  the rejected proposal was saved to ${ctx.config.state.proposalPath}`));
-      throw new UserError(err.message, "try a stronger synthesis model, or raise --budget / --max-edits");
+      printSynthesisFailure(err, ctx.config.state);
+      throw new UserError(err.message, synthesisFailureHint(err));
     }
     throw err;
   }

package/src/commands/run.js

@@ -2,7 +2,7 @@ import { UserError, color, info, json, out } from "../logger.js";
 import { ProposalViolation } from "../proposal.js";
 import { runAnalysis } from "./analyze.js";
 import { bootstrapJson, bootstrapRun, printBootstrap } from "./bootstrap.js";
-import { printProposal, runProposal } from "./propose.js";
+import { printProposal, printSynthesisFailure, runProposal, synthesisFailureHint } from "./propose.js";
 import { budgetBar, formatTokens } from "../tokens.js";
 import { startTui } from "../tui/index.js";
 import { resolveMemoryFiles } from "../memory.js";
@@ -10,7 +10,7 @@ import { resolveMemoryFiles } from "../memory.js";
 /**
  * The default command: one full backward pass.
  *
- *   discover -> distill -> analyze (cheap, fanned out) -> fold -> synthesize (one big call)
+ *   discover -> distill -> analyze (cheap, fanned out) -> fold -> synthesize (high-reasoning turns)
  *
  * It never writes - with one exception: a repo with no memory file at all is
  * bootstrapped (`./bootstrap.js`), which only ever creates files. Otherwise applying
@@ -31,6 +31,8 @@ export async function cmdRun(ctx) {
       )}`,
     );
 
+    config.state.clearProposal();
+
     if (!resolveMemoryFiles(repo.root, config.memoryFiles).primary) {
       const result = await bootstrapRun(ctx);
       tui?.stop();
@@ -82,11 +84,8 @@ export async function cmdRun(ctx) {
   } catch (err) {
     tui?.stop();
     if (err instanceof ProposalViolation) {
-      info("");
-      for (const violation of err.violations) info(`  ${color.red("x")} ${violation}`);
-      info("");
-      info(color.dim(`  the rejected proposal was saved to ${config.state.proposalPath}`));
-      throw new UserError(err.message, "try a stronger synthesis model, or raise --budget / --max-edits");
+      printSynthesisFailure(err, config.state);
+      throw new UserError(err.message, synthesisFailureHint(err));
     }
     throw err;
   } finally {