backpass 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +406 -0
  3. package/bin/backpass.js +4 -0
  4. package/package.json +62 -0
  5. package/src/acpx.js +576 -0
  6. package/src/agents.js +389 -0
  7. package/src/analyze.js +289 -0
  8. package/src/apply/lavish.js +128 -0
  9. package/src/apply/terminal.js +119 -0
  10. package/src/apply/writer.js +101 -0
  11. package/src/bootstrap.js +74 -0
  12. package/src/cli.js +261 -0
  13. package/src/commands/analyze.js +88 -0
  14. package/src/commands/apply.js +103 -0
  15. package/src/commands/bootstrap.js +172 -0
  16. package/src/commands/init.js +59 -0
  17. package/src/commands/propose.js +136 -0
  18. package/src/commands/run.js +95 -0
  19. package/src/commands/scan.js +90 -0
  20. package/src/commands/status.js +143 -0
  21. package/src/commands/usage.js +25 -0
  22. package/src/config.js +249 -0
  23. package/src/diff.js +305 -0
  24. package/src/discovery/adapters/claude.js +77 -0
  25. package/src/discovery/adapters/codex.js +162 -0
  26. package/src/discovery/adapters/cursor-cli.js +109 -0
  27. package/src/discovery/adapters/cursor-ide.js +130 -0
  28. package/src/discovery/adapters/grok.js +107 -0
  29. package/src/discovery/adapters/opencode.js +151 -0
  30. package/src/discovery/adapters/pi.js +87 -0
  31. package/src/discovery/adapters/shared.js +195 -0
  32. package/src/discovery/adapters/sqlite.js +50 -0
  33. package/src/discovery/association.js +100 -0
  34. package/src/discovery/index.js +226 -0
  35. package/src/discovery/self.js +62 -0
  36. package/src/distill.js +182 -0
  37. package/src/fold.js +214 -0
  38. package/src/gap-ledger.js +174 -0
  39. package/src/logger.js +74 -0
  40. package/src/memory.js +244 -0
  41. package/src/progress.js +29 -0
  42. package/src/prompts/analysis.md +48 -0
  43. package/src/prompts/annotate.md +48 -0
  44. package/src/prompts/synthesis.md +98 -0
  45. package/src/prompts.js +36 -0
  46. package/src/proposal.js +430 -0
  47. package/src/redact.js +36 -0
  48. package/src/repo.js +118 -0
  49. package/src/sample.js +99 -0
  50. package/src/skills.js +207 -0
  51. package/src/state.js +202 -0
  52. package/src/subprocess.js +47 -0
  53. package/src/synthesize.js +287 -0
  54. package/src/tokens.js +48 -0
  55. package/src/tui/index.js +336 -0
  56. package/src/tui/render.js +487 -0
  57. package/src/tui/term.js +130 -0
  58. package/src/tui/theme.js +111 -0
  59. package/src/workspace.js +162 -0
  60. package/templates/apply.html +928 -0
package/src/fold.js ADDED
@@ -0,0 +1,214 @@
1
+ import { similarity } from "./memory.js";
2
+ import { GAP_SIMILARITY_THRESHOLD, gapSource } from "./gap-ledger.js";
3
+
4
+ /**
5
+ * Stage 2 of the pipeline (design section 3): fold per-transcript evidence into one
6
+ * compact summary. Entirely deterministic - no model involved.
7
+ *
8
+ * Three things happen here that the synthesis pass depends on:
9
+ *
10
+ * 1. Evidence is grouped by instruction id, giving per-instruction positive/negative
11
+ * counts and, crucially, `relevance` = sessions where the instruction drew any
12
+ * evidence / sessions analyzed. That ratio is what decides memory-file vs skill
13
+ * placement in section 7.
14
+ * 2. Near-duplicate gaps from different sessions are clustered, so "three sessions
15
+ * re-derived the db schema" arrives as one item with three quotes.
16
+ * 3. Gap clusters below `minGapEvidence` are dropped. Batch size > 1: one bad session
17
+ * never rewrites the weights. Sessions are counted across runs, not per run: when the
18
+ * caller passes `gapObservations` (the pruned gap ledger, `src/gap-ledger.js`) the
19
+ * clusters are built from those instead of this run's records, so a gap seen once now
20
+ * and once on a later run graduates. Without a ledger the records alone are used.
21
+ */
22
+
23
+ /**
24
+ * @param {object[]} evidenceRecords
25
+ * @param {{ minGapEvidence?: number, memoryFile?: object|null, gapObservations?: object[]|null }} [options]
26
+ */
27
+ export function foldEvidence(evidenceRecords, { minGapEvidence = 2, memoryFile = null, gapObservations = null } = {}) {
28
+ const usable = evidenceRecords.filter((e) => e && e.status === "ok");
29
+ const analyzedSessions = usable.length;
30
+
31
+ const instructions = new Map();
32
+ let positiveCount = 0;
33
+ let negativeCount = 0;
34
+ let usedRawCount = 0;
35
+
36
+ const touch = (id) => {
37
+ if (!instructions.has(id)) {
38
+ instructions.set(id, {
39
+ instruction: id,
40
+ positive: 0,
41
+ negative: 0,
42
+ sessions: new Set(),
43
+ quotes: [],
44
+ });
45
+ }
46
+ return instructions.get(id);
47
+ };
48
+
49
+ const recordObservations = [];
50
+ for (const record of usable) {
51
+ if (record.usedRawTranscript) usedRawCount += 1;
52
+ const source = gapSource(record.transcript);
53
+
54
+ for (const polarity of ["positive", "negative"]) {
55
+ for (const item of record[polarity] || []) {
56
+ const entry = touch(item.instruction);
57
+ entry[polarity] += 1;
58
+ entry.sessions.add(record.transcript.id);
59
+ entry.quotes.push({ polarity, text: item.quote, effect: item.effect, moment: item.moment, source });
60
+ if (polarity === "positive") positiveCount += 1;
61
+ else negativeCount += 1;
62
+ }
63
+ }
64
+
65
+ for (const gap of record.gaps || []) {
66
+ recordObservations.push({
67
+ proposedInstruction: gap.proposedInstruction,
68
+ mistake: gap.mistake,
69
+ quote: gap.quote,
70
+ recurrenceRisk: gap.recurrenceRisk,
71
+ source,
72
+ sessionId: record.transcript.id,
73
+ });
74
+ }
75
+ }
76
+
77
+ const gapClusters = clusterGapObservations(gapObservations ?? recordObservations);
78
+
79
+ // Instructions that exist in the file but drew no evidence at all are the strongest
80
+ // removal / extraction candidates, so they must appear in the summary too.
81
+ if (memoryFile) {
82
+ for (const unit of memoryFile.units) touch(unit.id);
83
+ }
84
+
85
+ const instructionRows = [...instructions.values()]
86
+ .map((entry) => {
87
+ const unit = memoryFile?.units.find((u) => u.id === entry.instruction) || null;
88
+ return {
89
+ instruction: entry.instruction,
90
+ positive: entry.positive,
91
+ negative: entry.negative,
92
+ sessions: entry.sessions.size,
93
+ relevance: analyzedSessions ? entry.sessions.size / analyzedSessions : 0,
94
+ tokens: unit?.tokens ?? null,
95
+ section: unit?.section ?? null,
96
+ known: Boolean(unit),
97
+ quotes: entry.quotes.slice(0, 6),
98
+ };
99
+ })
100
+ .sort((a, b) => b.negative - a.negative || b.sessions - a.sessions || a.instruction.localeCompare(b.instruction));
101
+
102
+ const gaps = gapClusters
103
+ .map((cluster) => ({
104
+ proposedInstruction: cluster.proposedInstruction,
105
+ sessions: cluster.sessions.size,
106
+ recurrenceRisk: highestRisk(cluster.items),
107
+ quotes: cluster.items.slice(0, 6).map((i) => ({ text: i.quote, effect: i.mistake, source: i.source })),
108
+ }))
109
+ .filter((cluster) => cluster.sessions >= minGapEvidence)
110
+ .sort((a, b) => b.sessions - a.sessions);
111
+
112
+ const droppedGapSingletons = gapClusters.length - gaps.length;
113
+
114
+ return {
115
+ version: 1,
116
+ generatedAt: new Date().toISOString(),
117
+ analyzedSessions,
118
+ totals: {
119
+ positive: positiveCount,
120
+ negative: negativeCount,
121
+ gapClusters: gaps.length,
122
+ droppedGapSingletons,
123
+ usedRawTranscript: usedRawCount,
124
+ },
125
+ instructions: instructionRows,
126
+ gaps,
127
+ };
128
+ }
129
+
130
+ /**
131
+ * Greedy similarity clustering over gap observations. A cluster counts each session once
132
+ * no matter how many observations it contributed (re-analysis, duplicate reports).
133
+ */
134
+ export function clusterGapObservations(observations) {
135
+ const clusters = [];
136
+ for (const obs of observations) {
137
+ if (!obs || !obs.proposedInstruction) continue;
138
+ const cluster = clusters.find(
139
+ (c) => similarity(c.proposedInstruction, obs.proposedInstruction) >= GAP_SIMILARITY_THRESHOLD,
140
+ );
141
+ const item = {
142
+ mistake: obs.mistake,
143
+ quote: obs.quote,
144
+ recurrenceRisk: obs.recurrenceRisk,
145
+ source: obs.source,
146
+ sessionId: obs.sessionId,
147
+ };
148
+ if (cluster) {
149
+ if (!cluster.sessions.has(obs.sessionId)) cluster.items.push(item);
150
+ cluster.sessions.add(obs.sessionId);
151
+ // Keep the shortest phrasing: it generalizes best.
152
+ if (obs.proposedInstruction.length < cluster.proposedInstruction.length) {
153
+ cluster.proposedInstruction = obs.proposedInstruction;
154
+ }
155
+ } else {
156
+ clusters.push({
157
+ proposedInstruction: obs.proposedInstruction,
158
+ sessions: new Set([obs.sessionId]),
159
+ items: [item],
160
+ });
161
+ }
162
+ }
163
+ return clusters;
164
+ }
165
+
166
+ function highestRisk(items) {
167
+ const order = { high: 3, medium: 2, low: 1 };
168
+ return items.reduce((best, i) => (order[i.recurrenceRisk] > order[best] ? i.recurrenceRisk : best), "low");
169
+ }
170
+
171
+ /** Compact rendering of the folded evidence for the synthesis prompt. */
172
+ export function renderEvidenceForPrompt(summary) {
173
+ const lines = [];
174
+
175
+ lines.push(`Sessions analyzed: ${summary.analyzedSessions}`);
176
+ lines.push(
177
+ `Totals: ${summary.totals.positive} positive, ${summary.totals.negative} negative, ` +
178
+ `${summary.totals.gapClusters} gap clusters (${summary.totals.droppedGapSingletons} singletons dropped below threshold)`,
179
+ );
180
+ lines.push("");
181
+ lines.push("### Per-instruction evidence");
182
+ for (const row of summary.instructions) {
183
+ const relevance = `${(row.relevance * 100).toFixed(1)}%`;
184
+ const cost = row.tokens === null ? "" : ` cost=${row.tokens}tok`;
185
+ lines.push(
186
+ `- [${row.instruction}] +${row.positive} -${row.negative} sessions=${row.sessions} relevance=${relevance}${cost}` +
187
+ (row.known ? "" : " (id not found in current file - stale reference)"),
188
+ );
189
+ for (const quote of row.quotes.slice(0, 3)) {
190
+ lines.push(` ${quote.polarity === "negative" ? "-" : "+"} "${oneLine(quote.text)}" (${quote.source})`);
191
+ }
192
+ }
193
+
194
+ lines.push("");
195
+ lines.push("### Gap clusters (mistakes no current instruction covers)");
196
+ if (!summary.gaps.length) {
197
+ lines.push("- none above the evidence threshold");
198
+ }
199
+ for (const gap of summary.gaps) {
200
+ lines.push(`- sessions=${gap.sessions} risk=${gap.recurrenceRisk} :: ${gap.proposedInstruction}`);
201
+ for (const quote of gap.quotes.slice(0, 3)) {
202
+ lines.push(` "${oneLine(quote.text)}" (${quote.source})`);
203
+ }
204
+ }
205
+
206
+ return lines.join("\n");
207
+ }
208
+
209
+ function oneLine(text) {
210
+ const flat = String(text || "")
211
+ .replace(/\s+/g, " ")
212
+ .trim();
213
+ return flat.length > 240 ? `${flat.slice(0, 240)}...` : flat;
214
+ }
@@ -0,0 +1,174 @@
1
+ import { similarity } from "./memory.js";
2
+ import { parseSince } from "./config.js";
3
+ import { sha256 } from "./state.js";
4
+
5
+ /**
6
+ * Durable gap corroboration across runs (`.backpass/gap-ledger.json`).
7
+ *
8
+ * The fold stage only promotes a gap once `minGapEvidence` distinct sessions report it.
9
+ * Counting those sessions from the evidence that happens to be on disk is lossy: a
10
+ * transcript's evidence file is rewritten every time it is re-analyzed against a changed
11
+ * memory file (every apply changes the hash), and the analysis model rephrases gaps
12
+ * between runs, so two observations of one gap rarely line up in a single fold. This
13
+ * ledger keeps every gap observation, keyed by gap identity and session, so a gap seen in
14
+ * one session now and in another session on a later run still reaches the bar.
15
+ *
16
+ * Identity and freshness rules:
17
+ *
18
+ * - A gap's identity is its proposed instruction, matched by the same bigram similarity
19
+ * the in-run clustering uses (`GAP_SIMILARITY_THRESHOLD`), against the ledger's
20
+ * canonical phrasing (the shortest seen). The entry id is a hash of the first phrasing
21
+ * and never changes, so rephrasing does not split an entry.
22
+ * - Sessions are keyed by transcript id (harness + native session id), so re-analyzing
23
+ * or re-sampling the same session overwrites its observation and never adds a count.
24
+ * - A gap is a fact about its session: re-analysis that no longer mentions it is model
25
+ * noise, not the session changing, so observations are only ever replaced, not removed
26
+ * by absence. They retire in exactly two ways: the memory file gains an instruction
27
+ * that covers the gap (`GAP_COVERED_THRESHOLD`, the `reanchor` bar), or the sighting
28
+ * has waited longer than `gapLedgerMaxAge` for a partner, counted from when backpass
29
+ * first saw it (re-analysis never refreshes that clock; session age itself is already
30
+ * bounded by discovery's `since` at entry). Keying to the memory hash instead would
31
+ * reset the count on every unrelated edit, which is the failure this ledger fixes.
32
+ * - The file is fail-soft: a missing or corrupt ledger is rebuilt from this run's
33
+ * evidence, which is exactly what the pre-ledger fold saw.
34
+ */
35
+
36
+ /** Two gap phrasings at or above this Sorensen-Dice bigram score are one gap. */
37
+ export const GAP_SIMILARITY_THRESHOLD = 0.45;
38
+ /** A memory-file instruction this similar to a gap's proposal covers it. */
39
+ export const GAP_COVERED_THRESHOLD = 0.6;
40
+
41
+ export function emptyGapLedger() {
42
+ return { version: 1, entries: {} };
43
+ }
44
+
45
+ export function gapSource(transcript = {}) {
46
+ const date = transcript.startedAt ? new Date(transcript.startedAt).toISOString().slice(0, 10) : "unknown date";
47
+ return `${transcript.harness} · ${String(transcript.id || "")
48
+ .replace(/^[a-z-]+-/, "")
49
+ .slice(0, 8)} · ${date}`;
50
+ }
51
+
52
+ function normalize(text) {
53
+ return String(text || "")
54
+ .toLowerCase()
55
+ .replace(/[^a-z0-9\s]/g, " ")
56
+ .replace(/\s+/g, " ")
57
+ .trim();
58
+ }
59
+
60
+ export function gapEntryId(memoryPath, proposedInstruction) {
61
+ return sha256(`${memoryPath}\n${normalize(proposedInstruction)}`).slice(0, 16);
62
+ }
63
+
64
+ /** The ledger entry a proposed instruction belongs to, or null. */
65
+ export function findGapEntry(ledger, memoryPath, proposedInstruction) {
66
+ let best = null;
67
+ let bestScore = 0;
68
+ for (const entry of Object.values(ledger.entries)) {
69
+ if (entry.memoryPath !== memoryPath) continue;
70
+ const score = similarity(entry.proposedInstruction, proposedInstruction);
71
+ if (score >= GAP_SIMILARITY_THRESHOLD && score > bestScore) {
72
+ best = entry;
73
+ bestScore = score;
74
+ }
75
+ }
76
+ return best;
77
+ }
78
+
79
+ /**
80
+ * Fold this run's evidence into the ledger. One observation per (gap, session); a
81
+ * session seen again replaces its own observation and keeps its first-seen timestamp.
82
+ */
83
+ export function recordGapObservations(ledger, evidenceRecords, { now = new Date() } = {}) {
84
+ const observedAt = new Date(now).toISOString();
85
+ let recorded = 0;
86
+ for (const record of evidenceRecords) {
87
+ if (!record || record.status !== "ok" || !record.memoryPath) continue;
88
+ const transcript = record.transcript || {};
89
+ if (!transcript.id) continue;
90
+ for (const gap of record.gaps || []) {
91
+ if (!gap || !gap.proposedInstruction) continue;
92
+ let entry = findGapEntry(ledger, record.memoryPath, gap.proposedInstruction);
93
+ if (!entry) {
94
+ const id = gapEntryId(record.memoryPath, gap.proposedInstruction);
95
+ entry = ledger.entries[id] = {
96
+ id,
97
+ memoryPath: record.memoryPath,
98
+ proposedInstruction: gap.proposedInstruction,
99
+ sessions: {},
100
+ };
101
+ } else if (gap.proposedInstruction.length < entry.proposedInstruction.length) {
102
+ // Keep the shortest phrasing: it generalizes best (same rule as the in-run fold).
103
+ entry.proposedInstruction = gap.proposedInstruction;
104
+ }
105
+ const prior = entry.sessions[transcript.id];
106
+ entry.sessions[transcript.id] = {
107
+ firstObservedAt: prior?.firstObservedAt || observedAt,
108
+ observedAt,
109
+ sessionStartedAt: transcript.startedAt ?? prior?.sessionStartedAt ?? null,
110
+ memoryHash: record.memoryHash || null,
111
+ source: gapSource(transcript),
112
+ mistake: gap.mistake,
113
+ quote: gap.quote,
114
+ recurrenceRisk: gap.recurrenceRisk,
115
+ };
116
+ recorded += 1;
117
+ }
118
+ }
119
+ return recorded;
120
+ }
121
+
122
+ /**
123
+ * Retire observations that no longer count: sightings first seen more than `maxAge` ago
124
+ * (a duration like `90d`, `all` to disable) and gaps the current memory file now covers.
125
+ */
126
+ export function pruneGapLedger(
127
+ ledger,
128
+ { memoryFile = null, memoryPath = null, maxAge = "90d", now = new Date() } = {},
129
+ ) {
130
+ const maxAgeMs = parseSince(maxAge);
131
+ const cutoff = maxAgeMs === null ? -Infinity : new Date(now).getTime() - maxAgeMs;
132
+ const stats = { expired: 0, covered: 0 };
133
+
134
+ for (const [id, entry] of Object.entries(ledger.entries)) {
135
+ const applies = memoryPath === null || entry.memoryPath === memoryPath;
136
+ if (applies && memoryFile && isCovered(memoryFile, entry.proposedInstruction)) {
137
+ stats.covered += Object.keys(entry.sessions).length;
138
+ delete ledger.entries[id];
139
+ continue;
140
+ }
141
+ for (const [sessionId, obs] of Object.entries(entry.sessions)) {
142
+ const when = Date.parse(obs.firstObservedAt || obs.observedAt);
143
+ if (!Number.isFinite(when) || when < cutoff) {
144
+ stats.expired += 1;
145
+ delete entry.sessions[sessionId];
146
+ }
147
+ }
148
+ if (!Object.keys(entry.sessions).length) delete ledger.entries[id];
149
+ }
150
+ return stats;
151
+ }
152
+
153
+ function isCovered(memoryFile, proposedInstruction) {
154
+ return (memoryFile.units || []).some((unit) => similarity(unit.text, proposedInstruction) >= GAP_COVERED_THRESHOLD);
155
+ }
156
+
157
+ /** Flatten the ledger into the observation list `foldEvidence` clusters over. */
158
+ export function ledgerGapObservations(ledger, memoryPath) {
159
+ const observations = [];
160
+ for (const entry of Object.values(ledger.entries)) {
161
+ if (entry.memoryPath !== memoryPath) continue;
162
+ for (const [sessionId, obs] of Object.entries(entry.sessions)) {
163
+ observations.push({
164
+ proposedInstruction: entry.proposedInstruction,
165
+ sessionId,
166
+ source: obs.source,
167
+ mistake: obs.mistake,
168
+ quote: obs.quote,
169
+ recurrenceRisk: obs.recurrenceRisk,
170
+ });
171
+ }
172
+ }
173
+ return observations;
174
+ }
package/src/logger.js ADDED
@@ -0,0 +1,74 @@
1
+ const NO_COLOR = process.env.NO_COLOR !== undefined || !process.stderr.isTTY;
2
+ const CSI = "\u001b[";
3
+
4
+ const wrap = (code) => (s) => (NO_COLOR ? String(s) : `${CSI}${code}m${s}${CSI}0m`);
5
+
6
+ export const color = {
7
+ dim: wrap("2"),
8
+ bold: wrap("1"),
9
+ red: wrap("31"),
10
+ green: wrap("32"),
11
+ yellow: wrap("33"),
12
+ blue: wrap("34"),
13
+ cyan: wrap("36"),
14
+ };
15
+
16
+ let quiet = false;
17
+ let sink = null;
18
+
19
+ export function setQuiet(value) {
20
+ quiet = Boolean(value);
21
+ }
22
+
23
+ /**
24
+ * While the live progress view owns stderr, progress lines are diverted here,
25
+ * buffered, and replayed verbatim on teardown - so scrollback after a TUI run
26
+ * is byte-identical to a run without one. Pass null to restore direct output.
27
+ */
28
+ export function setLoggerSink(fn) {
29
+ sink = fn;
30
+ }
31
+
32
+ /** Human-facing progress and diagnostics go to stderr so stdout stays pipeable. */
33
+ export function info(...args) {
34
+ if (quiet) return;
35
+ if (sink) {
36
+ sink(args.join(" "));
37
+ return;
38
+ }
39
+ console.error(...args);
40
+ }
41
+
42
+ export function step(label, detail = "") {
43
+ info(`${color.cyan("·")} ${label}${detail ? ` ${color.dim(detail)}` : ""}`);
44
+ }
45
+
46
+ export function warn(message) {
47
+ const line = `${color.yellow("warn")} ${message}`;
48
+ if (sink) {
49
+ sink(line);
50
+ return;
51
+ }
52
+ console.error(line);
53
+ }
54
+
55
+ export function fail(message) {
56
+ console.error(`${color.red("error")} ${message}`);
57
+ }
58
+
59
+ /** Structured results go to stdout. */
60
+ export function out(text) {
61
+ console.log(text);
62
+ }
63
+
64
+ export function json(value) {
65
+ console.log(JSON.stringify(value, null, 2));
66
+ }
67
+
68
+ export class UserError extends Error {
69
+ constructor(message, hint) {
70
+ super(message);
71
+ this.name = "UserError";
72
+ this.hint = hint;
73
+ }
74
+ }
package/src/memory.js ADDED
@@ -0,0 +1,244 @@
1
+ import fs from "node:fs";
2
+ import path from "node:path";
3
+
4
+ import { sha256 } from "./state.js";
5
+ import { estimateTokens } from "./tokens.js";
6
+
7
+ /**
8
+ * Memory files are the weights. To talk about them precisely, backpass parses each
9
+ * file into addressable instruction units (design section 3.1):
10
+ *
11
+ * - sections come from markdown headings
12
+ * - units come from list items and paragraphs inside those sections
13
+ * - each unit gets a content hash (survives cosmetic edits elsewhere in the file)
14
+ * and a readable alias AG-001, AG-002, ... used in prompts and evidence
15
+ *
16
+ * Evidence anchors to the alias; the hash is what lets us re-anchor across runs.
17
+ */
18
+
19
+ const FENCE = /^\s*(```|~~~)/;
20
+
21
+ function normalizeForHash(text) {
22
+ return text
23
+ .toLowerCase()
24
+ .replace(/[`*_~]/g, "")
25
+ .replace(/\s+/g, " ")
26
+ .trim();
27
+ }
28
+
29
+ export function unitHash(text) {
30
+ return sha256(normalizeForHash(text)).slice(0, 12);
31
+ }
32
+
33
+ function alias(index) {
34
+ return `AG-${String(index + 1).padStart(3, "0")}`;
35
+ }
36
+
37
+ /**
38
+ * Split memory-file text into instruction units. Fenced code blocks stay attached to
39
+ * the paragraph they belong to rather than being split into nonsense lines.
40
+ */
41
+ export function parseMemoryUnits(text) {
42
+ const lines = text.split("\n");
43
+ const units = [];
44
+ const headings = [];
45
+
46
+ let buffer = [];
47
+ let bufferStart = 0;
48
+ let inFence = false;
49
+ let fenceMarker = null;
50
+
51
+ const flush = (endLine) => {
52
+ const raw = buffer.join("\n");
53
+ if (raw.trim()) {
54
+ units.push({
55
+ text: raw.replace(/\s+$/, ""),
56
+ section: headings.join(" > "),
57
+ startLine: bufferStart + 1,
58
+ endLine,
59
+ });
60
+ }
61
+ buffer = [];
62
+ };
63
+
64
+ for (let i = 0; i < lines.length; i += 1) {
65
+ const line = lines[i];
66
+
67
+ if (FENCE.test(line)) {
68
+ const marker = line.trim().slice(0, 3);
69
+ if (!inFence) {
70
+ inFence = true;
71
+ fenceMarker = marker;
72
+ } else if (marker === fenceMarker) {
73
+ inFence = false;
74
+ }
75
+ if (!buffer.length) bufferStart = i;
76
+ buffer.push(line);
77
+ continue;
78
+ }
79
+
80
+ if (inFence) {
81
+ buffer.push(line);
82
+ continue;
83
+ }
84
+
85
+ const heading = line.match(/^(#{1,6})\s+(.*)$/);
86
+ if (heading) {
87
+ flush(i);
88
+ const depth = heading[1].length;
89
+ headings.length = Math.min(headings.length, depth - 1);
90
+ headings[depth - 1] = heading[2].trim();
91
+ for (let d = 0; d < depth - 1; d += 1) headings[d] = headings[d] ?? "";
92
+ continue;
93
+ }
94
+
95
+ const isListItem = /^\s*([-*+]|\d+[.)])\s+/.test(line);
96
+ const isBlank = line.trim() === "";
97
+
98
+ if (isBlank) {
99
+ flush(i);
100
+ continue;
101
+ }
102
+ // A new list item starts a new unit; continuation lines stay with it.
103
+ if (isListItem && buffer.length && /^\s*([-*+]|\d+[.)])\s+/.test(buffer[0])) {
104
+ flush(i);
105
+ }
106
+ if (!buffer.length) bufferStart = i;
107
+ buffer.push(line);
108
+ }
109
+ flush(lines.length);
110
+
111
+ return units.map((unit, index) => ({
112
+ id: alias(index),
113
+ hash: unitHash(unit.text),
114
+ tokens: estimateTokens(unit.text),
115
+ ...unit,
116
+ }));
117
+ }
118
+
119
+ export function readMemoryFile(repoRoot, relativePath) {
120
+ const absolute = path.join(repoRoot, relativePath);
121
+ if (!fs.existsSync(absolute)) return null;
122
+ const text = fs.readFileSync(absolute, "utf8");
123
+ return {
124
+ path: relativePath,
125
+ absolute,
126
+ text,
127
+ hash: `sha256:${sha256(text).slice(0, 16)}`,
128
+ tokens: estimateTokens(text),
129
+ units: parseMemoryUnits(text),
130
+ };
131
+ }
132
+
133
+ /** Load every configured memory file that actually exists in the repo. */
134
+ export function loadMemoryFiles(repoRoot, memoryFiles) {
135
+ return memoryFiles.map((f) => readMemoryFile(repoRoot, f)).filter(Boolean);
136
+ }
137
+
138
+ /** Combined hash across all memory files - the "weights version" evidence is keyed to. */
139
+ export function memorySetHash(files) {
140
+ return `sha256:${sha256(files.map((f) => `${f.path}:${f.hash}`).join("|")).slice(0, 16)}`;
141
+ }
142
+
143
+ /**
144
+ * Render the instruction index that both prompt tiers see. It is a lookup table keyed
145
+ * by alias, never a stand-in for the file: units are listed with the lines they occupy
146
+ * so the synthesis agent can find them in the raw file it edits.
147
+ */
148
+ export function renderInstructionIndex(file) {
149
+ return file.units
150
+ .map((u) => {
151
+ const lines = u.startLine === u.endLine ? `L${u.startLine}` : `L${u.startLine}-${u.endLine}`;
152
+ return `[${u.id}] (${u.tokens} tok, ${lines})${u.section ? ` <${u.section}>` : ""}\n${u.text}`;
153
+ })
154
+ .join("\n\n");
155
+ }
156
+
157
+ function bigrams(text) {
158
+ const words = normalizeForHash(text).split(" ").filter(Boolean);
159
+ if (words.length < 2) return new Set(words);
160
+ const out = new Set();
161
+ for (let i = 0; i < words.length - 1; i += 1) out.add(`${words[i]} ${words[i + 1]}`);
162
+ return out;
163
+ }
164
+
165
+ /** Sorensen-Dice similarity over word bigrams, used for re-anchoring. */
166
+ export function similarity(a, b) {
167
+ const A = bigrams(a);
168
+ const B = bigrams(b);
169
+ if (!A.size || !B.size) return A.size === B.size ? 1 : 0;
170
+ let shared = 0;
171
+ for (const g of A) if (B.has(g)) shared += 1;
172
+ return (2 * shared) / (A.size + B.size);
173
+ }
174
+
175
+ /**
176
+ * Re-anchor an instruction reference from a previous run onto the current file.
177
+ * Exact hash match wins; otherwise the best fuzzy match above threshold; otherwise stale.
178
+ */
179
+ export function reanchor(reference, file, threshold = 0.6) {
180
+ if (reference.hash) {
181
+ const exact = file.units.find((u) => u.hash === reference.hash);
182
+ if (exact) return { unit: exact, match: "hash", score: 1 };
183
+ }
184
+ if (reference.id) {
185
+ const byId = file.units.find((u) => u.id === reference.id);
186
+ if (byId && (!reference.text || similarity(byId.text, reference.text) >= threshold)) {
187
+ return { unit: byId, match: "id", score: 1 };
188
+ }
189
+ }
190
+ if (!reference.text) return { unit: null, match: "stale", score: 0 };
191
+
192
+ let best = null;
193
+ let bestScore = 0;
194
+ for (const unit of file.units) {
195
+ const score = similarity(unit.text, reference.text);
196
+ if (score > bestScore) {
197
+ best = unit;
198
+ bestScore = score;
199
+ }
200
+ }
201
+ if (best && bestScore >= threshold) return { unit: best, match: "fuzzy", score: bestScore };
202
+ return { unit: null, match: "stale", score: bestScore };
203
+ }
204
+
205
+ /**
206
+ * Pointer detection. The convention for covering both harness families without
207
+ * duplicating content is a canonical AGENTS.md plus a CLAUDE.md that only contains the
208
+ * `@AGENTS.md` import (Claude Code inlines it at load time). Such a file carries no
209
+ * instructions of its own, so optimizing the target is fully correct and the pointer
210
+ * stays valid afterwards.
211
+ *
212
+ * A file is a pointer to `target` when, ignoring blank lines and HTML comments, its
213
+ * only content is the import line (`@AGENTS.md` or `@./AGENTS.md`).
214
+ */
215
+ export function isPointerTo(text, target) {
216
+ const lines = text
217
+ .replace(/<!--[\s\S]*?-->/g, "")
218
+ .split("\n")
219
+ .map((l) => l.trim())
220
+ .filter(Boolean);
221
+ if (lines.length !== 1) return false;
222
+ const ref = lines[0].replace(/^@\.\//, "@");
223
+ return ref === `@${target}`;
224
+ }
225
+
226
+ /**
227
+ * Resolve the memory file a run optimizes from the configured order.
228
+ *
229
+ * primary the first configured file that exists (null when none does)
230
+ * pointers other configured files that are pointers to the primary - silently fine
231
+ * separate other configured files with their own content - NOT updated by a run;
232
+ * the caller warns so divergence is never silent
233
+ *
234
+ * The weights hash still covers every existing file, as before, so cached evidence
235
+ * survives this resolution unchanged.
236
+ */
237
+ export function resolveMemoryFiles(repoRoot, memoryFiles) {
238
+ const files = loadMemoryFiles(repoRoot, memoryFiles);
239
+ if (!files.length) return { primary: null, all: files, pointers: [], separate: [], hash: null };
240
+ const [primary, ...others] = files;
241
+ const pointers = others.filter((f) => isPointerTo(f.text, primary.path));
242
+ const separate = others.filter((f) => !pointers.includes(f));
243
+ return { primary, all: files, pointers, separate, hash: memorySetHash(files) };
244
+ }