backpass 0.1.11 → 0.1.13

package/src/sample.js

@@ -1,5 +1,8 @@
+import crypto from "node:crypto";
+
 import { parseMaxTranscripts, parseSince } from "./config.js";
 import { color, info } from "./logger.js";
+import { transcriptIdentity } from "./transcript.js";
 
 /**
  * Recency-weighted capping of the discovered transcript set.
@@ -14,7 +17,22 @@ import { color, info } from "./logger.js";
  * Sampling uses the Efraimidis-Spirakis one-pass scheme: key = -ln(u) / weight with
  * u ~ U(0, 1), keep the N smallest keys. That is exactly weighted sampling without
  * replacement (an exponential race with rate = weight), needs no rejection loop, and is
- * O(n log n). The RNG is a seedable splitmix32 so runs are reproducible with `--seed`.
+ * O(n log n).
+ *
+ * `u` is NOT drawn from a shared PRNG stream stepped once per transcript - that would
+ * make every transcript's draw depend on how many other transcripts came before it in
+ * the array, so an unrelated insertion or a reordering would reshuffle everyone's draw
+ * (and, with `config.seed` defaulting to null, the CLI reseeded from `Math.random()` on
+ * every invocation, so even an unchanged rerun drew a different sample - see the
+ * `sample-reuse` regression tests). Instead each transcript's `u` is `sampleUnit`, a hash
+ * of that transcript's canonical discovery identity, never its array position, plus the
+ * configured seed. That makes sampling: (1)
+ * deterministic and sticky by default, with no persisted state - the same corpus and
+ * config always draw the same `u` per transcript; (2) stable under growth - a transcript
+ * already in the corpus keeps the exact same `u` (and so the same key, modulo its own
+ * weight) when other transcripts are added, removed, or reordered; new transcripts just
+ * compete for slots on the same footing. `--seed` still selects a different, equally
+ * reproducible hash input.
  *
  * This module is pure: it never touches the cache, so evidence for a sampled transcript
  * is reused by the analyzer exactly as before.
@@ -22,18 +40,19 @@ import { color, info } from "./logger.js";
 
 export const DEFAULT_SAMPLE_HALF_LIFE = "14d";
 
-/** splitmix32: small, seedable, and uniform enough for sampling keys. */
-export function seededRandom(seed) {
-  let state = Number(seed) >>> 0 || 0x9e3779b9;
-  return () => {
-    state = (state + 0x9e3779b9) | 0;
-    let t = state ^ (state >>> 16);
-    t = Math.imul(t, 0x21f0aaad);
-    t ^= t >>> 15;
-    t = Math.imul(t, 0x735a2d97);
-    t ^= t >>> 15;
-    return (t >>> 0) / 4294967296;
-  };
+/**
+ * Deterministic draw in [0, 1) for one transcript: a SHA-256 of its durable identity
+ * and `seed` (or a fixed default when unseeded), so it depends only on that transcript and
+ * the configured seed - never on discovery order, the cap, or which other
+ * transcripts are present. Discovery removes duplicate canonical identities, while
+ * distinct identities that happen to draw equal keys are ordered by identity.
+ */
+export function sampleUnit(transcript, seed) {
+  const digest = crypto
+    .createHash("sha256")
+    .update(`${seed ?? "default"} ${transcriptIdentity(transcript)}`, "utf8")
+    .digest();
+  return digest.readUInt32BE(0) / 0x100000000;
 }
 
 /** Epoch ms a transcript is dated to: the session start, else the file mtime. */
@@ -67,13 +86,16 @@ export function sampleTranscripts(
   { seed, now = Date.now(), halfLife = DEFAULT_SAMPLE_HALF_LIFE } = {},
 ) {
   if (count === null || transcripts.length <= count) return transcripts;
-  const random = seededRandom(seed ?? Math.floor(Math.random() * 0xffffffff));
   const halfLifeMs = parseSince(halfLife) ?? Infinity;
   const keyed = transcripts.map((transcript, index) => {
-    const u = random() || Number.EPSILON;
-    return { index, key: -Math.log(u) / recencyWeight(transcript, { now, halfLifeMs }) };
+    const u = sampleUnit(transcript, seed) || Number.EPSILON;
+    return {
+      index,
+      identity: transcriptIdentity(transcript),
+      key: -Math.log(u) / recencyWeight(transcript, { now, halfLifeMs }),
+    };
   });
-  keyed.sort((a, b) => a.key - b.key);
+  keyed.sort((a, b) => a.key - b.key || (a.identity < b.identity ? -1 : a.identity > b.identity ? 1 : 0));
   const kept = new Set(keyed.slice(0, count).map((k) => k.index));
   return transcripts.filter((_, index) => kept.has(index));
 }

package/src/state.js

@@ -5,6 +5,7 @@ import crypto from "node:crypto";
 import { STATE_DIRNAME } from "./config.js";
 import { warn } from "./logger.js";
 import { ensureLocalExclude } from "./repo.js";
+import { transcriptIdentity } from "./transcript.js";
 
 /** The line every command writes to the repo's local git exclude for the state dir. */
 export const STATE_EXCLUDE_LINE = `${STATE_DIRNAME}/`;
@@ -16,7 +17,7 @@ export const STATE_EXCLUDE_LINE = `${STATE_DIRNAME}/`;
  * the tracked `.gitignore`:
  *
  *   scan-cache.json        path+mtime+size -> association verdict (design section 2.2)
- *   evidence/<id>.json     per-transcript tier-1 analysis output (design section 3)
+ *   evidence/<identity>.json per-transcript tier-1 analysis output (design section 3)
  *   evidence-summary.json  folded evidence (stage 2)
  *   proposal.json          latest parseable tier-2 synthesis; absent if none was produced (stage 3)
  *   rejections.json        edits the human rejected, and the evidence weight behind them
@@ -76,25 +77,58 @@ export class State {
     this.writeJsonFile(this.scanCachePath, cache);
   }
 
-  evidencePath(transcriptId) {
-    return path.join(this.evidenceDir, `${safeFileName(transcriptId)}.json`);
+  evidencePath(transcript) {
+    const identity = transcript && typeof transcript === "object" ? transcriptIdentity(transcript) : transcript;
+    return path.join(this.evidenceDir, `${safeFileName(identity)}.json`);
   }
 
-  readEvidence(transcriptId) {
-    return this.readJsonFile(this.evidencePath(transcriptId), null);
+  readEvidence(transcript) {
+    const currentPath = this.evidencePath(transcript);
+    const current = this.readJsonFile(currentPath, null);
+    if (!transcript || typeof transcript !== "object") return current;
+
+    const identity = transcriptIdentity(transcript);
+    const legacyPath = this.evidencePath(transcript.id);
+    if (legacyPath === currentPath) return current;
+    const legacy = this.readJsonFile(legacyPath, null);
+    const legacyMatches = legacy?.transcript && transcriptIdentity(legacy.transcript) === identity;
+    if (current) {
+      const currentMatches = current.transcript && transcriptIdentity(current.transcript) === identity;
+      const migrated = currentMatches ? migrateEvidenceRecord(current, transcript, identity) : current;
+      if (migrated !== current) this.writeJsonFile(currentPath, migrated);
+      if (legacyMatches) fs.rmSync(legacyPath, { force: true });
+      return migrated;
+    }
+    if (!legacyMatches) return null;
+
+    const migrated = migrateEvidenceRecord(legacy, transcript, identity);
+    // Publish the upgraded record atomically before removing the legacy path. If the
+    // process stops between these operations, the next read prefers the valid canonical
+    // record and merely cleans up the duplicate.
+    this.writeJsonFile(currentPath, migrated);
+    fs.rmSync(legacyPath, { force: true });
+    return migrated;
   }
 
-  writeEvidence(transcriptId, evidence) {
-    this.writeJsonFile(this.evidencePath(transcriptId), evidence);
+  writeEvidence(transcript, evidence) {
+    this.writeJsonFile(this.evidencePath(transcript), evidence);
   }
 
   listEvidence() {
     if (!fs.existsSync(this.evidenceDir)) return [];
-    return fs
+    const records = fs
       .readdirSync(this.evidenceDir)
-      .filter((f) => f.endsWith(".json"))
-      .map((f) => this.readJsonFile(path.join(this.evidenceDir, f), null))
-      .filter(Boolean);
+      .filter((file) => file.endsWith(".json"))
+      .map((file) => ({ file, record: this.readJsonFile(path.join(this.evidenceDir, file), null) }))
+      .filter(({ record }) => Boolean(record));
+    const unique = new Map();
+    for (const item of records) {
+      const identity = item.record.transcript ? transcriptIdentity(item.record.transcript) : `file:${item.file}`;
+      const canonical = item.record.transcript ? path.basename(this.evidencePath(item.record.transcript)) : item.file;
+      const existing = unique.get(identity);
+      if (!existing || item.file === canonical) unique.set(identity, item);
+    }
+    return [...unique.values()].map(({ record }) => record);
   }
 
   readSummary() {
@@ -158,12 +192,23 @@ export function sha256(text) {
   return crypto.createHash("sha256").update(text, "utf8").digest("hex");
 }
 
+function migrateEvidenceRecord(record, transcript, identity) {
+  const legacyKey = `${transcript.mtimeMs}:${transcript.bytes}:${record.memoryHash}`;
+  const key = record.key === legacyKey ? evidenceKey(transcript, record.memoryHash) : record.key;
+  if (record.transcript?.identity === identity && record.key === key) return record;
+  return {
+    ...record,
+    transcript: { ...record.transcript, identity },
+    key,
+  };
+}
+
 /**
  * Cache key for a transcript's analysis: the transcript's own content signature plus
  * the memory-file hash it was judged against. Either changing invalidates the evidence.
  */
 export function evidenceKey(transcript, memoryHash) {
-  return `${transcript.mtimeMs}:${transcript.bytes}:${memoryHash}`;
+  return `${transcriptIdentity(transcript)}:${transcript.mtimeMs}:${transcript.bytes}:${memoryHash}`;
 }
 
 /**

package/src/synthesize.js

@@ -70,10 +70,12 @@ function budgetRule(memoryFile, config, maxEdits) {
       `This file is ALREADY ${Math.abs(remaining)} tokens OVER budget, so this run is a SHRINK ` +
       `PLAN. You are NOT expected to reach ${config.budgetTokens} tokens in one run - the ` +
       `${maxEdits}-edit cap for this run makes that impossible and later runs continue the work. ` +
-      `What is required is real progress: the edit set MUST be net-negative, so lead with the ` +
-      `highest-cost instructions that have no positive evidence, and with skill extractions of ` +
-      `long narrow sections. Any addition must name the removal that pays for it. Make the ` +
-      `largest honest reduction you can justify from the evidence.`
+      `What is required is real progress: the edit set MUST be net-negative, so lead with skill ` +
+      `extractions of long, narrow, crisply-triggered sections - extraction frees the same ` +
+      `always-loaded tokens and loses nothing, and it never needs removal evidence. Deleting an ` +
+      `instruction outright still needs its harm-evidence floor; the budget never lowers that ` +
+      `bar. Any addition must name the removal or extraction that pays for it. Make the largest ` +
+      `honest reduction you can justify from the evidence.`
     );
   }
   if (remaining < config.budgetTokens * 0.15) {

package/src/transcript.js

@@ -0,0 +1,15 @@
+import crypto from "node:crypto";
+
+export function transcriptIdentity(transcript) {
+  if (typeof transcript?.identity === "string" && transcript.identity) return transcript.identity;
+  const harness = String(transcript?.harness ?? "");
+  let nativeId = String(transcript?.nativeId ?? transcript?.id ?? "");
+  if (transcript?.nativeId == null && harness && nativeId.startsWith(`${harness}-`)) {
+    nativeId = nativeId.slice(harness.length + 1);
+  }
+  const source = String(transcript?.path ?? "");
+  return crypto
+    .createHash("sha256")
+    .update(JSON.stringify([harness, nativeId, source]), "utf8")
+    .digest("hex");
+}

package/src/workspace.js

@@ -1,7 +1,8 @@
 import fs from "node:fs";
 import path from "node:path";
 
-import { anchoredHunks } from "./diff.js";
+import { anchoredHunks, countOccurrences, span } from "./diff.js";
+import { parseMemoryUnits } from "./memory.js";
 import { parseFrontmatter } from "./skills.js";
 import { sha256 } from "./state.js";
 
@@ -94,6 +95,112 @@ export function parseSkillFile(relative, text) {
   };
 }
 
+/**
+ * One line's identity for extraction-recovery checks: verbatim modulo the noise a
+ * faithful move is allowed to make. Unicode dashes fold to "-" (house style normalizes
+ * them during moves) and interior whitespace collapses; anything more is a real change.
+ */
+export function normalizeRecoveryLine(line) {
+  return String(line ?? "")
+    .replace(/[‐-―−]/g, "-")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+/** The normalized non-blank lines of a set of file texts, with occurrence counts. */
+export function recoveredLineCounts(texts) {
+  const counts = new Map();
+  for (const text of texts) {
+    for (const line of String(text ?? "").split("\n")) {
+      const normalized = normalizeRecoveryLine(line);
+      if (normalized) counts.set(normalized, (counts.get(normalized) || 0) + 1);
+    }
+  }
+  return counts;
+}
+
+/**
+ * Split one pure-removal memory-file hunk at the boundary between text that lands in a
+ * created skill and text that vanishes. Adjacent removals merge into one measured change
+ * (`anchoredHunks`), so without this split an extraction and an unrelated deletion that
+ * happen to sit next to each other in the file fuse into a single accept/reject decision.
+ * The boundary is a decision boundary, and it falls on instruction-unit edges because a
+ * skill carries whole sections.
+ *
+ * Returns the sub-hunks, or null when there is nothing to split (one kind only) or the
+ * sub-hunks cannot be given unique, non-overlapping spans, in which case the merged hunk
+ * is kept instead.
+ */
+export function splitRemovalHunk(hunk, { oldText, oldLines, recovered }) {
+  // A removal that reaches the file's true tail cannot be split into independent
+  // sub-hunks: `span`'s tail rule gives the final sub-hunk a LEADING newline, and that
+  // is the same character its predecessor owns as its trailing newline. Applying one
+  // decision then consumes the separator the other one's `find` needs, so the pair
+  // composes in only one order - which breaks the any-subset-any-order contract the
+  // writer relies on. No non-overlapping anchoring exists at that seam; keep the
+  // merged hunk instead (a file ending in "\n" is unaffected: its final split("\n")
+  // element is the empty string, which no text removal reaches).
+  if (hunk.oldEnd === oldLines.length) return null;
+
+  const lineKinds = new Map();
+  for (let lineNo = hunk.oldStart; lineNo <= hunk.oldEnd; lineNo += 1) {
+    const normalized = normalizeRecoveryLine(oldLines[lineNo - 1]);
+    if (!normalized) continue;
+    const remaining = recovered.get(normalized) || 0;
+    lineKinds.set(lineNo, remaining > 0 ? "recovered" : "deleted");
+    if (remaining > 0) recovered.set(normalized, remaining - 1);
+  }
+
+  for (const unit of parseMemoryUnits(oldText)) {
+    const start = Math.max(unit.startLine, hunk.oldStart);
+    const end = Math.min(unit.endLine, hunk.oldEnd);
+    if (start > end) continue;
+    const kinds = new Set();
+    for (let lineNo = start; lineNo <= end; lineNo += 1) {
+      if (lineKinds.has(lineNo)) kinds.add(lineKinds.get(lineNo));
+    }
+    if (kinds.size > 1) return null;
+  }
+
+  for (let lineNo = hunk.oldStart; lineNo <= hunk.oldEnd; lineNo += 1) {
+    if (!/^#{1,6}\s+/.test(oldLines[lineNo - 1] || "")) continue;
+    let contentLine = lineNo + 1;
+    while (contentLine <= hunk.oldEnd && !normalizeRecoveryLine(oldLines[contentLine - 1])) contentLine += 1;
+    if (contentLine > hunk.oldEnd || /^#{1,6}\s+/.test(oldLines[contentLine - 1] || "")) continue;
+    if (lineKinds.get(lineNo) !== lineKinds.get(contentLine)) return null;
+  }
+
+  // Group the removed lines into maximal runs by recovery; blank lines never start a
+  // run and attach to whichever run surrounds them.
+  const runs = [];
+  for (let lineNo = hunk.oldStart; lineNo <= hunk.oldEnd; lineNo += 1) {
+    const kind = lineKinds.get(lineNo);
+    if (!kind) continue;
+    const current = runs[runs.length - 1];
+    if (current && current.kind === kind) current.last = lineNo;
+    else runs.push({ kind, first: lineNo, last: lineNo });
+  }
+  if (runs.length < 2) return null;
+
+  const subHunks = runs.map((run, index) => {
+    const start = index === 0 ? hunk.oldStart : runs[index - 1].last + 1;
+    const end = index === runs.length - 1 ? hunk.oldEnd : run.last;
+    const find = span(oldLines, start - 1, end);
+    return {
+      find,
+      replace: "",
+      oldStart: start,
+      oldEnd: end,
+      removed: end - start + 1,
+      added: 0,
+      lines: oldLines.slice(start - 1, end).map((text) => ({ type: "del", text })),
+    };
+  });
+
+  if (subHunks.some((sub) => !sub.find || countOccurrences(oldText, sub.find) !== 1)) return null;
+  return subHunks;
+}
+
 /**
  * Everything that differs between the originals and the workspace now, as changes with
  * stable ids the annotate turn refers to:
@@ -138,6 +245,27 @@ export function measureWorkspace(workspace) {
     changes.push({ kind: "created", file: relative, text, skill: parseSkillFile(relative, text) });
   }
 
+  // With the created files known, split any memory-file removal that mixes extracted
+  // text (recovered in a created file) with deleted text, so the deletion is its own
+  // measured change and stays independently decidable.
+  const createdTexts = changes.filter((c) => c.kind === "created").map((c) => c.text);
+  if (createdTexts.length) {
+    const recovered = recoveredLineCounts(createdTexts);
+    const oldText = originals.get(memoryPath) ?? "";
+    const oldLines = oldText.split("\n");
+    const measured = [];
+    for (const change of changes) {
+      if (change.kind !== "hunk" || change.file !== memoryPath || !change.removed || change.added) {
+        measured.push(change);
+        continue;
+      }
+      const subHunks = splitRemovalHunk(change, { oldText, oldLines, recovered });
+      if (subHunks) measured.push(...subHunks.map((sub) => ({ kind: "hunk", file: memoryPath, ...sub })));
+      else measured.push(change);
+    }
+    changes.splice(0, changes.length, ...measured);
+  }
+
   changes.forEach((change, index) => {
     change.id = `H${index + 1}`;
   });