@mmerterden/multi-agent-pipeline 20.33.0 → 20.34.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/CHANGELOG.md +43 -0
  2. package/README.md +2 -1
  3. package/README.tr.md +2 -1
  4. package/docs/adr/0018-cross-repo-links-not-a-global-graph.md +43 -0
  5. package/docs/adr/README.md +2 -1
  6. package/docs/facts.json +5 -4
  7. package/docs/features.md +1 -1
  8. package/install/_dev-only-files.mjs +1 -0
  9. package/install/catalog-history.json +1 -1
  10. package/install/claude.mjs +9 -2
  11. package/install/templates/claude-hooks.json +12 -1
  12. package/manifest.json +98 -75
  13. package/package.json +3 -1
  14. package/pipeline/agents/code-reviewer.md +9 -0
  15. package/pipeline/agents/explorer.md +4 -0
  16. package/pipeline/agents/plan-critic.md +4 -2
  17. package/pipeline/commands/multi-agent/SKILL.md +17 -136
  18. package/pipeline/commands/multi-agent/channels/SKILL.md +15 -297
  19. package/pipeline/commands/multi-agent/graph/SKILL.md +5 -2
  20. package/pipeline/commands/multi-agent/language/SKILL.md +3 -3
  21. package/pipeline/commands/multi-agent/resume/SKILL.md +8 -22
  22. package/pipeline/commands/multi-agent/review/SKILL.md +7 -131
  23. package/pipeline/commands/multi-agent/setup/SKILL.md +19 -224
  24. package/pipeline/contract/README.md +5 -2
  25. package/pipeline/multi-agent-refs/channels/generator.md +146 -0
  26. package/pipeline/multi-agent-refs/channels/menu.md +55 -0
  27. package/pipeline/multi-agent-refs/channels/multi-repo.md +6 -0
  28. package/pipeline/multi-agent-refs/channels/preview.md +25 -0
  29. package/pipeline/multi-agent-refs/channels/reference.md +98 -0
  30. package/pipeline/multi-agent-refs/features/autopilot-circuit-breaker.md +1 -1
  31. package/pipeline/multi-agent-refs/features/code-graph.md +27 -0
  32. package/pipeline/multi-agent-refs/features/office-threat-model.md +1 -1
  33. package/pipeline/multi-agent-refs/features/plan-critic.md +5 -2
  34. package/pipeline/multi-agent-refs/features/review-delta.md +4 -1
  35. package/pipeline/multi-agent-refs/features/scope-check.md +10 -1
  36. package/pipeline/multi-agent-refs/knowledge.md +9 -0
  37. package/pipeline/multi-agent-refs/orchestrator/dispatcher.md +163 -0
  38. package/pipeline/multi-agent-refs/phases/phase-1-plan.md +4 -4
  39. package/pipeline/multi-agent-refs/phases/phase-2-dev.md +9 -8
  40. package/pipeline/multi-agent-refs/phases/phase-3-review.md +12 -7
  41. package/pipeline/multi-agent-refs/phases/phase-4-commit.md +7 -3
  42. package/pipeline/multi-agent-refs/phases/phase-5-report.md +6 -3
  43. package/pipeline/multi-agent-refs/picker-contract.md +1 -1
  44. package/pipeline/multi-agent-refs/resume/details.md +35 -0
  45. package/pipeline/multi-agent-refs/review/inputs.md +87 -0
  46. package/pipeline/multi-agent-refs/review/module-guides.md +25 -0
  47. package/pipeline/multi-agent-refs/review/valven.md +42 -0
  48. package/pipeline/multi-agent-refs/setup/accounts.md +27 -0
  49. package/pipeline/multi-agent-refs/setup/app-store-connect.md +5 -0
  50. package/pipeline/multi-agent-refs/setup/firebase.md +5 -0
  51. package/pipeline/multi-agent-refs/setup/google-play.md +4 -0
  52. package/pipeline/multi-agent-refs/setup/hooks-and-plugins.md +21 -0
  53. package/pipeline/multi-agent-refs/setup/language.md +46 -0
  54. package/pipeline/multi-agent-refs/setup/multi-repo-group.md +51 -0
  55. package/pipeline/multi-agent-refs/setup/summaries.md +63 -0
  56. package/pipeline/multi-agent-refs/setup/token-save-flow.md +51 -0
  57. package/pipeline/office/index.html +12 -7
  58. package/pipeline/office/toolkit.json +1 -1
  59. package/pipeline/rules/response-shape.md +6 -7
  60. package/pipeline/schemas/agent-state.schema.json +18 -0
  61. package/pipeline/schemas/phases.json +4 -4
  62. package/pipeline/schemas/plan-critique.schema.json +9 -2
  63. package/pipeline/schemas/planning-output.schema.json +25 -2
  64. package/pipeline/schemas/prefs.schema.json +24 -1
  65. package/pipeline/schemas/reviewer-output.schema.json +20 -2
  66. package/pipeline/schemas/token-budget.json +4 -4
  67. package/pipeline/schemas/triage-output.schema.json +20 -2
  68. package/pipeline/scripts/_graph-analysis.mjs +270 -0
  69. package/pipeline/scripts/_memory-policy.mjs +165 -0
  70. package/pipeline/scripts/_secret-scan.mjs +105 -0
  71. package/pipeline/scripts/_unified-diff.mjs +2 -2
  72. package/pipeline/scripts/capture-flush.sh +12 -2
  73. package/pipeline/scripts/check-read-size.sh +9 -0
  74. package/pipeline/scripts/code-graph-rules/android.json +8 -3
  75. package/pipeline/scripts/contract-server.mjs +36 -15
  76. package/pipeline/scripts/debt-ledger.mjs +323 -0
  77. package/pipeline/scripts/file-context.mjs +261 -0
  78. package/pipeline/scripts/file-context.sh +20 -0
  79. package/pipeline/scripts/graph-links.mjs +230 -0
  80. package/pipeline/scripts/graph-query.mjs +133 -1
  81. package/pipeline/scripts/graph-report.mjs +65 -0
  82. package/pipeline/scripts/learn-from-transcripts.mjs +12 -48
  83. package/pipeline/scripts/learnings-ledger.mjs +143 -24
  84. package/pipeline/scripts/plan-critique-gate.mjs +11 -2
  85. package/pipeline/scripts/rework-rulings.mjs +106 -0
  86. package/pipeline/scripts/scope-check-gate.mjs +149 -2
  87. package/pipeline/scripts/triage-memory.mjs +64 -37
  88. package/pipeline/scripts/validate-planning.mjs +68 -4
  89. package/pipeline/scripts/validate-reviewer.mjs +20 -0
  90. package/pipeline/scripts/validate-triage.mjs +107 -2
  91. package/pipeline/skills/.skill-manifest.json +5 -5
  92. package/pipeline/skills/shared/core/multi-agent-graph/SKILL.md +5 -2
  93. package/pipeline/skills/shared/core/multi-agent-language/SKILL.md +4 -4
  94. package/pipeline/skills/shared/core/multi-agent-review/SKILL.md +2 -0
  95. package/pipeline/skills/shared/core/multi-agent-setup/SKILL.md +4 -4
@@ -0,0 +1,270 @@
1
+ /**
2
+ * @file _graph-analysis.mjs - whole-graph views over the code graph.
3
+ *
4
+ * File-level structure the per-symbol queries cannot show:
5
+ *
6
+ * importCycles files that import each other, directly or around a loop
7
+ * (strongly connected components of the file import graph)
8
+ * clusters groups of files more connected to each other than to the
9
+ * rest, by label propagation over imports and references,
10
+ * with a cohesion score (internal / all edges touching it)
11
+ * graphDiff what changed between two builds of the same repository
12
+ * questions a short list of questions worth asking before a change,
13
+ * built from hubs, cycles and the edges between clusters
14
+ *
15
+ * Deterministic: same graph in, same answer out, so a report diff means the
16
+ * code changed. Zero dependencies; the algorithms are small enough to own.
17
+ *
18
+ * @module pipeline/scripts/_graph-analysis
19
+ */
20
+
21
+ const LP_ROUNDS = 20;
22
+
23
+ function fileOfSymbolMap(graph) {
24
+ const fileOf = new Map();
25
+ for (const e of graph.edges) {
26
+ if (e.kind === "defines" && e.from.startsWith("file:")) fileOf.set(e.to, e.from);
27
+ }
28
+ return fileOf;
29
+ }
30
+
31
+ /**
32
+ * Directed file -> file edges: an import of a file, or a reference to a symbol
33
+ * another file defines. External modules are left out.
34
+ *
35
+ * @returns {Map<string, Map<string, {imports: number, references: number}>>}
36
+ */
37
+ export function fileEdges(graph) {
38
+ const files = new Set(graph.nodes.filter((n) => n.kind === "file").map((n) => n.id));
39
+ const fileOf = fileOfSymbolMap(graph);
40
+ const out = new Map([...files].map((f) => [f, new Map()]));
41
+ const add = (from, to, kind) => {
42
+ if (!from || !to || from === to || !files.has(from) || !files.has(to)) return;
43
+ const row = out.get(from).get(to) ?? { imports: 0, references: 0 };
44
+ row[kind]++;
45
+ out.get(from).set(to, row);
46
+ };
47
+ for (const e of graph.edges) {
48
+ const from = files.has(e.from) ? e.from : fileOf.get(e.from);
49
+ if (e.kind === "imports") add(from, e.to, "imports");
50
+ else if (e.kind === "references" || e.kind === "inherits") {
51
+ add(from, files.has(e.to) ? e.to : fileOf.get(e.to), "references");
52
+ }
53
+ }
54
+ return out;
55
+ }
56
+
57
+ /**
58
+ * Import cycles: strongly connected components of size two or more of the
59
+ * file import graph, largest first. Iterative Tarjan, so a deep repository
60
+ * cannot overflow the stack.
61
+ *
62
+ * @returns {string[][]}
63
+ */
64
+ export function importCycles(graph) {
65
+ const edges = fileEdges(graph);
66
+ const succ = (f) => [...edges.get(f)].filter(([, w]) => w.imports > 0).map(([t]) => t);
67
+ const index = new Map();
68
+ const low = new Map();
69
+ const onStack = new Set();
70
+ const stack = [];
71
+ const found = [];
72
+ let next = 0;
73
+ for (const root of [...edges.keys()].sort()) {
74
+ if (index.has(root)) continue;
75
+ const work = [{ id: root, it: 0, nbrs: null }];
76
+ while (work.length) {
77
+ const top = work[work.length - 1];
78
+ if (top.nbrs === null) {
79
+ index.set(top.id, next);
80
+ low.set(top.id, next);
81
+ next++;
82
+ stack.push(top.id);
83
+ onStack.add(top.id);
84
+ top.nbrs = succ(top.id).sort();
85
+ }
86
+ if (top.it < top.nbrs.length) {
87
+ const w = top.nbrs[top.it++];
88
+ if (!index.has(w)) work.push({ id: w, it: 0, nbrs: null });
89
+ else if (onStack.has(w)) low.set(top.id, Math.min(low.get(top.id), index.get(w)));
90
+ continue;
91
+ }
92
+ work.pop();
93
+ if (work.length) {
94
+ const parent = work[work.length - 1].id;
95
+ low.set(parent, Math.min(low.get(parent), low.get(top.id)));
96
+ }
97
+ if (low.get(top.id) === index.get(top.id)) {
98
+ const members = [];
99
+ let m;
100
+ do {
101
+ m = stack.pop();
102
+ onStack.delete(m);
103
+ members.push(m);
104
+ } while (m !== top.id);
105
+ if (members.length > 1) found.push(members.sort());
106
+ }
107
+ }
108
+ }
109
+ return found.sort((a, b) => b.length - a.length || a[0].localeCompare(b[0]));
110
+ }
111
+
112
+ /**
113
+ * Clusters by label propagation on the undirected, weighted file graph. Each
114
+ * round every file takes the label its neighbours carry the most weight for;
115
+ * among equal labels the smallest wins, a file keeps its own label on a tie,
116
+ * and files are visited in id order, so the result does not depend on chance.
117
+ *
118
+ * The cluster's id is its most connected file; `dirs` names where its files sit.
119
+ *
120
+ * @returns {Array<{id: string, dirs: string, files: string[], cohesion: number, internal: number, external: number}>}
121
+ */
122
+ export function clusters(graph, { rounds = LP_ROUNDS, minSize = 2 } = {}) {
123
+ const directed = fileEdges(graph);
124
+ const nbrs = new Map([...directed.keys()].map((f) => [f, new Map()]));
125
+ for (const [from, row] of directed) {
126
+ for (const [to, w] of row) {
127
+ const weight = w.imports + w.references;
128
+ nbrs.get(from).set(to, (nbrs.get(from).get(to) ?? 0) + weight);
129
+ nbrs.get(to).set(from, (nbrs.get(to).get(from) ?? 0) + weight);
130
+ }
131
+ }
132
+ const order = [...nbrs.keys()].sort();
133
+ const label = new Map(order.map((f) => [f, f]));
134
+ for (let r = 0; r < rounds; r++) {
135
+ let changed = false;
136
+ for (const f of order) {
137
+ const tally = new Map();
138
+ for (const [n, w] of nbrs.get(f)) tally.set(label.get(n), (tally.get(label.get(n)) ?? 0) + w);
139
+ if (!tally.size) continue;
140
+ let best = null;
141
+ let bestW = -1;
142
+ for (const [l, w] of [...tally].sort((a, b) => a[0].localeCompare(b[0]))) {
143
+ if (w > bestW) {
144
+ best = l;
145
+ bestW = w;
146
+ }
147
+ }
148
+ // A file keeps its label on a tie, so one bridge edge cannot pull a
149
+ // whole group across.
150
+ if (best !== label.get(f) && bestW > (tally.get(label.get(f)) ?? 0)) {
151
+ label.set(f, best);
152
+ changed = true;
153
+ }
154
+ }
155
+ if (!changed) break;
156
+ }
157
+ const groups = new Map();
158
+ for (const f of order) {
159
+ const l = label.get(f);
160
+ if (!groups.has(l)) groups.set(l, []);
161
+ groups.get(l).push(f);
162
+ }
163
+ const result = [];
164
+ for (const files of groups.values()) {
165
+ if (files.length < minSize) continue;
166
+ const set = new Set(files);
167
+ let internal = 0;
168
+ let external = 0;
169
+ for (const f of files) {
170
+ for (const [n, w] of nbrs.get(f)) {
171
+ if (set.has(n)) internal += w;
172
+ else external += w;
173
+ }
174
+ }
175
+ internal /= 2;
176
+ const total = internal + external;
177
+ const hub = files
178
+ .map((f) => [f, [...nbrs.get(f)].reduce((sum, [n, w]) => sum + (set.has(n) ? w : 0), 0)])
179
+ .sort((x, y) => y[1] - x[1] || x[0].localeCompare(y[0]))[0][0];
180
+ result.push({
181
+ id: hub.replace(/^file:/, ""),
182
+ dirs: commonPrefix(files),
183
+ files,
184
+ internal,
185
+ external,
186
+ cohesion: total ? Math.round((internal / total) * 100) / 100 : 1,
187
+ });
188
+ }
189
+ return result.sort((a, b) => b.files.length - a.files.length || a.id.localeCompare(b.id));
190
+ }
191
+
192
+ /** A cluster's name: the directory most of its files sit in, and how many others. */
193
+ function commonPrefix(files) {
194
+ const dirs = new Map();
195
+ for (const f of files) {
196
+ const path = f.replace(/^file:/, "");
197
+ const dir = path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : ".";
198
+ dirs.set(dir, (dirs.get(dir) ?? 0) + 1);
199
+ }
200
+ const ranked = [...dirs].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
201
+ return ranked.length > 1 ? `${ranked[0][0]} (+${ranked.length - 1} dirs)` : ranked[0][0];
202
+ }
203
+
204
+ /** Nodes and edges added and removed between two builds. */
205
+ export function graphDiff(before, after) {
206
+ const ids = (g, kind) => new Set(g.nodes.filter((n) => n.kind === kind).map((n) => n.id));
207
+ const edgeKey = (e) => `${e.from}|${e.kind}|${e.to}`;
208
+ const delta = (a, b) => ({
209
+ added: [...b].filter((x) => !a.has(x)).sort(),
210
+ removed: [...a].filter((x) => !b.has(x)).sort(),
211
+ });
212
+ const edgesA = new Set(before.edges.map(edgeKey));
213
+ const edgesB = new Set(after.edges.map(edgeKey));
214
+ const e = delta(edgesA, edgesB);
215
+ return {
216
+ files: delta(ids(before, "file"), ids(after, "file")),
217
+ symbols: delta(ids(before, "symbol"), ids(after, "symbol")),
218
+ edges: { added: e.added.length, removed: e.removed.length },
219
+ cycles: {
220
+ before: importCycles(before).length,
221
+ after: importCycles(after).length,
222
+ },
223
+ };
224
+ }
225
+
226
+ /**
227
+ * Questions worth asking before a change, from what the graph can show:
228
+ * the widest hubs, each import cycle, and the strongest link between two
229
+ * clusters. Templates only; no model call.
230
+ */
231
+ export function questions(graph, { hubs = [], cycles = null, groups = null, max = 8 } = {}) {
232
+ const out = [];
233
+ const shortPath = (id) => id.replace(/^file:/, "");
234
+ for (const h of hubs.slice(0, 3)) {
235
+ out.push(`What depends on \`${h.name}\` (${h.degree} edges), and which of it has tests?`);
236
+ }
237
+ for (const c of (cycles ?? importCycles(graph)).slice(0, 2)) {
238
+ const shown = c.slice(0, 3).map(shortPath).join("`, `");
239
+ out.push(
240
+ `Why do \`${shown}\`${c.length > 3 ? ` and ${c.length - 3} more` : ""} import each other?`,
241
+ );
242
+ }
243
+ const list = groups ?? clusters(graph);
244
+ const clusterOf = new Map();
245
+ list.forEach((g, i) => g.files.forEach((f) => clusterOf.set(f, i)));
246
+ const cross = new Map();
247
+ for (const [from, row] of fileEdges(graph)) {
248
+ for (const [to, w] of row) {
249
+ const a = clusterOf.get(from);
250
+ const b = clusterOf.get(to);
251
+ if (a === undefined || b === undefined || a === b) continue;
252
+ const key = `${a}>${b}`;
253
+ const cur = cross.get(key) ?? { a, b, weight: 0, via: from };
254
+ cur.weight += w.imports + w.references;
255
+ cross.set(key, cur);
256
+ }
257
+ }
258
+ const strongest = [...cross.values()].sort((x, y) => y.weight - x.weight).slice(0, 2);
259
+ for (const l of strongest) {
260
+ out.push(
261
+ `Should the cluster around \`${list[l.a].id}\` reach into the one around \`${list[l.b].id}\` directly (${l.weight} edges, e.g. from \`${shortPath(l.via)}\`)?`,
262
+ );
263
+ }
264
+ for (const g of list.filter((x) => x.files.length >= 3 && x.cohesion < 0.3).slice(0, 2)) {
265
+ out.push(
266
+ `The cluster around \`${g.id}\` (${g.dirs}) has cohesion ${g.cohesion}: one module, or several sharing a folder?`,
267
+ );
268
+ }
269
+ return out.slice(0, max);
270
+ }
@@ -0,0 +1,165 @@
1
+ /**
2
+ * @file _memory-policy.mjs - what the per-repo memory stores may keep, and
3
+ * how much weight an entry has earned.
4
+ *
5
+ * ledgerRowId(row) the stable id of a learnings-ledger row, the one
6
+ * feedback is recorded against
7
+ * checkedSlug(slug) the slug as one folder name; a separator, NUL, `.`
8
+ * or `..`, anything that could leave
9
+ * ~/.claude/memory/multi-agent/, throws
10
+ * memoryExcluded(slug) prefs.global.memory.excludeProjects lists repo slugs
11
+ * whose memory is never written; every store asks
12
+ * before it writes
13
+ * standing(feedback) what the outcomes recorded against one ledger entry
14
+ * say about it: suppressed (marked corrected, or
15
+ * judged a dead end more often than useful, twice or
16
+ * more), corroborated (useful at least twice) and a
17
+ * strength that decays with age (ACT-R base level,
18
+ * ln of the sum of age^-0.5 over the useful dates)
19
+ * nearDuplicate(a, b) two statements that say the same thing in other
20
+ * words: TF-IDF cosine at or above the threshold, the
21
+ * same numbers, the same negations and mostly the
22
+ * same word order ("A over B" is not "B over A")
23
+ *
24
+ * Zero dependencies.
25
+ *
26
+ * @module pipeline/scripts/_memory-policy
27
+ */
28
+
29
+ import { readFileSync } from "node:fs";
30
+ import { homedir } from "node:os";
31
+ import { join } from "node:path";
32
+
33
+ import { shortId, tokens } from "./_retrieval.mjs";
34
+
35
+ const CORROBORATION = 2;
36
+ const NEAR_DUP = 0.9;
37
+ const ORDER_OVERLAP = 0.5;
38
+ const NEGATIONS = /\b(not|never|no|none|without|except|only|avoid|don't|do not)\b/gi;
39
+
40
+ function prefs(env) {
41
+ const p =
42
+ env.MULTI_AGENT_PREFS || join(env.HOME || homedir(), ".claude", "multi-agent-preferences.json");
43
+ try {
44
+ return JSON.parse(readFileSync(p, "utf8"));
45
+ } catch {
46
+ return {};
47
+ }
48
+ }
49
+
50
+ /** Stable id of a ledger row: its kind and its whitespace-folded statement. */
51
+ export function ledgerRowId(row) {
52
+ const statement = String(row?.statement || "")
53
+ .toLowerCase()
54
+ .replace(/\s+/g, " ")
55
+ .trim();
56
+ return shortId("L", row?.kind, statement);
57
+ }
58
+
59
+ /**
60
+ * The slug, when it names one folder; throws on anything that could leave the
61
+ * memory folder (a separator, NUL, `.` or `..`). Spaces and non-ASCII letters
62
+ * are fine: a repository folder may carry them.
63
+ */
64
+ export function checkedSlug(slug) {
65
+ const s = String(slug ?? "");
66
+ if (!s || s.length > 255 || /[/\\\0]/.test(s) || s === "." || s === "..") {
67
+ throw new Error(`not a repository slug: ${JSON.stringify(s.slice(0, 80))}`);
68
+ }
69
+ return s.normalize("NFC");
70
+ }
71
+
72
+ /** True when the repo's memory must not be written. */
73
+ export function memoryExcluded(slug, env = process.env) {
74
+ const list = prefs(env)?.global?.memory?.excludeProjects;
75
+ const want = String(slug ?? "").normalize("NFC");
76
+ return Array.isArray(list) && list.some((p) => String(p).normalize("NFC") === want);
77
+ }
78
+
79
+ /**
80
+ * @param {Array<{ts: string, outcome: string}>} feedback rows for one entry
81
+ * @returns {{useful: number, deadEnd: number, corrected: number, suppressed: boolean, corroborated: boolean, strength: number}}
82
+ */
83
+ export function standing(feedback, now = Date.now()) {
84
+ const useful = feedback.filter((f) => f.outcome === "useful");
85
+ const deadEnd = feedback.filter((f) => f.outcome === "dead_end").length;
86
+ const corrected = feedback.filter((f) => f.outcome === "corrected").length;
87
+ const sum = useful.reduce((s, f) => {
88
+ const t = Date.parse(f.ts);
89
+ const days = Number.isNaN(t) ? 365 : Math.max(0, (now - t) / 86400000);
90
+ return s + Math.pow(days + 1, -0.5);
91
+ }, 0);
92
+ return {
93
+ useful: useful.length,
94
+ deadEnd,
95
+ corrected,
96
+ suppressed: corrected > 0 || (deadEnd >= CORROBORATION && deadEnd > useful.length),
97
+ corroborated: useful.length >= CORROBORATION,
98
+ strength: sum > 0 ? Math.log(sum) + 10 : 0,
99
+ };
100
+ }
101
+
102
+ const numbersIn = (s) => (String(s).match(/\d+(?:\.\d+)?/g) || []).sort().join(",");
103
+ const negationsIn = (s) =>
104
+ (String(s).toLowerCase().match(NEGATIONS) || [])
105
+ .map((w) => w.replace(/\s+/g, " "))
106
+ .sort()
107
+ .join(",");
108
+
109
+ /** Inverse document frequencies over a corpus of token lists. */
110
+ function idf(corpus) {
111
+ const df = new Map();
112
+ for (const doc of corpus) for (const t of new Set(doc)) df.set(t, (df.get(t) || 0) + 1);
113
+ const n = corpus.length;
114
+ return (t) => Math.log(1 + n / (df.get(t) || 1));
115
+ }
116
+
117
+ function tfidf(doc, weight) {
118
+ const tf = new Map();
119
+ for (const t of doc) tf.set(t, (tf.get(t) || 0) + 1);
120
+ const v = new Map();
121
+ for (const [t, c] of tf) v.set(t, c * weight(t));
122
+ return v;
123
+ }
124
+
125
+ function cosine(va, vb) {
126
+ let dot = 0;
127
+ for (const [t, w] of va) dot += w * (vb.get(t) || 0);
128
+ const norm = (v) => Math.sqrt([...v.values()].reduce((s, w) => s + w * w, 0));
129
+ const d = norm(va) * norm(vb);
130
+ return d ? dot / d : 0;
131
+ }
132
+
133
+ /** Jaccard overlap of adjacent token pairs: low when the same words come in another order. */
134
+ function orderOverlap(a, b) {
135
+ const pairs = (doc) => new Set(doc.slice(1).map((t, i) => `${doc[i]} ${t}`));
136
+ const pa = pairs(a);
137
+ const pb = pairs(b);
138
+ if (!pa.size || !pb.size) return 0;
139
+ let shared = 0;
140
+ for (const p of pa) if (pb.has(p)) shared++;
141
+ return shared / (pa.size + pb.size - shared);
142
+ }
143
+
144
+ /**
145
+ * The existing statement `candidate` repeats in other words, or null.
146
+ *
147
+ * @param {string} candidate
148
+ * @param {string[]} existing statements of the same kind
149
+ */
150
+ export function nearDuplicate(candidate, existing) {
151
+ const c = tokens(candidate);
152
+ if (c.length < 3) return null;
153
+ const docs = existing.map((s) => tokens(s));
154
+ const weight = idf([...docs, c]);
155
+ const vc = tfidf(c, weight);
156
+ const nums = numbersIn(candidate);
157
+ const negs = negationsIn(candidate);
158
+ for (let i = 0; i < existing.length; i++) {
159
+ if (numbersIn(existing[i]) !== nums) continue;
160
+ if (negationsIn(existing[i]) !== negs) continue;
161
+ if (orderOverlap(c, docs[i]) < ORDER_OVERLAP) continue;
162
+ if (cosine(vc, tfidf(docs[i], weight)) >= NEAR_DUP) return existing[i];
163
+ }
164
+ return null;
165
+ }
@@ -0,0 +1,105 @@
1
+ /**
2
+ * @file _secret-scan.mjs - the check every memory store runs before it writes.
3
+ *
4
+ * Text goes to a 0600 temp file and through `pre-commit-check.sh --scan-file`,
5
+ * which owns the secret patterns; a hit is refused, never written. A store
6
+ * that cannot find the scanner refuses as well, because "unchecked" and
7
+ * "clean" are different answers and only one of them may reach a file that
8
+ * later rides into a prompt.
9
+ *
10
+ * `<private>...</private>` spans are removed before the scan, so a caller can
11
+ * mark text that must not be remembered; an unclosed tag drops everything
12
+ * after it.
13
+ *
14
+ * Control bytes are removed from the scanned copy: the scanner skips a file
15
+ * `file --mime` calls binary, and one NUL is enough for that verdict. Removing
16
+ * them also rejoins a token split by one. A run that starts with a known token
17
+ * prefix is scanned a second time with its inner whitespace removed, so a token
18
+ * hard-wrapped by a space or a line break is still caught; prose without such
19
+ * a prefix is scanned once, as written.
20
+ *
21
+ * @module pipeline/scripts/_secret-scan
22
+ */
23
+
24
+ import { execFileSync } from "node:child_process";
25
+ import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
26
+ import { homedir, tmpdir } from "node:os";
27
+ import { dirname, join } from "node:path";
28
+ import { fileURLToPath } from "node:url";
29
+
30
+ const HERE = dirname(fileURLToPath(import.meta.url));
31
+ // eslint-disable-next-line no-control-regex
32
+ const CONTROL = /[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g;
33
+ const WRAPPED_TOKEN =
34
+ /(?:gh[pousr]_|github_pat_|glpat-|xox[baprs]-|sk-|sk_live_|rk_live_|npm_|hf_|gsk_|pplx-|fig[a-z]_|eyJ|AIza|AKIA|ASIA)[A-Za-z0-9_.-]{0,256}(?:\s+[A-Za-z0-9_.-]{1,256}){1,16}/g;
35
+
36
+ /** The text with whitespace removed inside every run that starts like a token. */
37
+ function rejoined(text) {
38
+ return text.replace(WRAPPED_TOKEN, (m) => m.replace(/\s+/g, ""));
39
+ }
40
+
41
+ /** Remove `<private>` spans; an unclosed opening tag drops the rest. */
42
+ export function stripPrivate(text) {
43
+ return String(text ?? "")
44
+ .replace(/<private>[\s\S]*?<\/private>/gi, " ")
45
+ .replace(/<private>[\s\S]*$/i, " ")
46
+ .replace(/<\/private>/gi, " ");
47
+ }
48
+
49
+ function scannerPath() {
50
+ const local = join(HERE, "pre-commit-check.sh");
51
+ if (existsSync(local)) return local;
52
+ const installed = join(homedir(), ".claude", "scripts", "pre-commit-check.sh");
53
+ return existsSync(installed) ? installed : null;
54
+ }
55
+
56
+ /**
57
+ * A scanner bound to one temp directory. `isClean(text)` is true only when
58
+ * the scanner ran and found nothing; call `close()` when done.
59
+ */
60
+ export function openScanner() {
61
+ const scanner = scannerPath();
62
+ if (!scanner) return { scanned: false, isClean: () => false, close() {} };
63
+ const dir = mkdtempSync(join(tmpdir(), "secret-scan-"));
64
+ const file = join(dir, "candidate.txt");
65
+ return {
66
+ scanned: true,
67
+ isClean(text) {
68
+ const plain = String(text ?? "").replace(CONTROL, "");
69
+ const copies = [plain];
70
+ const joined = rejoined(plain);
71
+ if (joined !== plain) copies.push(joined);
72
+ return copies.every((copy) => {
73
+ writeFileSync(file, `${copy}\n`, { mode: 0o600 });
74
+ try {
75
+ execFileSync("bash", [scanner, "--scan-file", file], {
76
+ stdio: "ignore",
77
+ timeout: 15_000,
78
+ });
79
+ return true;
80
+ } catch {
81
+ return false;
82
+ }
83
+ });
84
+ },
85
+ close() {
86
+ rmSync(dir, { recursive: true, force: true });
87
+ },
88
+ };
89
+ }
90
+
91
+ /** Split items into kept and dropped by the text `textOf(item)` returns. */
92
+ export function dropSecrets(items, textOf) {
93
+ const scan = openScanner();
94
+ const kept = [];
95
+ let dropped = 0;
96
+ try {
97
+ for (const it of items) {
98
+ if (scan.isClean(textOf(it))) kept.push(it);
99
+ else dropped++;
100
+ }
101
+ } finally {
102
+ scan.close();
103
+ }
104
+ return { kept, dropped, scanned: scan.scanned };
105
+ }
@@ -10,7 +10,7 @@
10
10
 
11
11
  /**
12
12
  * @typedef {{line: number, text: string}} DiffLine
13
- * @typedef {{removed: DiffLine[], added: DiffLine[]}} Hunk
13
+ * @typedef {{header: string, removed: DiffLine[], added: DiffLine[]}} Hunk
14
14
  * @typedef {{added: string[], addedAt: DiffLine[], removed: string[], removedAt: DiffLine[],
15
15
  * hunks: Hunk[], isNew: boolean, isDeleted: boolean, binary: boolean}} DiffFile
16
16
  */
@@ -73,7 +73,7 @@ export function parseUnifiedDiff(text) {
73
73
  let newLeft = h[4] === undefined ? 1 : Number(h[4]);
74
74
  let oldNo = Number(h[1]);
75
75
  let newNo = Number(h[3]);
76
- const hunk = { removed: [], added: [] };
76
+ const hunk = { header: h[0], removed: [], added: [] };
77
77
  cur.hunks.push(hunk);
78
78
  while ((oldLeft > 0 || newLeft > 0) && i + 1 < lines.length) {
79
79
  const body = lines[++i];
@@ -200,12 +200,22 @@ if [ -n "$TRIAGE" ]; then
200
200
  if [ -n "$TASK_ID" ]; then
201
201
  DISTILLED=$(run_store distill \
202
202
  node "$SCRIPTS/learnings-ledger.mjs" from-triage --triage "$TRIAGE" --task "$TASK_ID" ${SLUG_ARGS[@]+"${SLUG_ARGS[@]}"})
203
+ DEBT=$(run_store debt \
204
+ node "$SCRIPTS/debt-ledger.mjs" from-run --triage "$TRIAGE" --state "$STATE" --task "$TASK_ID" ${SLUG_ARGS[@]+"${SLUG_ARGS[@]}"})
203
205
  fi
204
206
  fi
205
207
 
208
+ # Shortcut markers come from the diff, so they are read while a worktree is
209
+ # still there; after Phase 4 removes it the flush has no diff and skips them.
210
+ BASE=$(jq -r '.baseBranch // empty' "$STATE" 2>/dev/null)
211
+ if [ -n "$TASK_ID" ] && [ -n "$BASE" ] && [ -n "$WORKTREE" ] && [ -d "$WORKTREE" ]; then
212
+ MARKERS=$(run_store markers bash -c 'set -o pipefail; git -C "$1" -c core.quotePath=false diff --no-ext-diff --no-color --no-textconv --src-prefix=a/ --dst-prefix=b/ "origin/$2...HEAD" | node "$3/debt-ledger.mjs" add --patch - --task "$4" "${@:5}"' \
213
+ bash "$WORKTREE" "$BASE" "$SCRIPTS" "$TASK_ID" ${SLUG_ARGS[@]+"${SLUG_ARGS[@]}"})
214
+ fi
215
+
206
216
  if [ "$JSON" -eq 1 ]; then
207
- printf '{"status":"flushed","taskId":"%s","triage":"%s","ingest":"%s","distill":"%s"}\n' \
208
- "$TASK_ID" "${TRIAGE:-none}" "$INGESTED" "$DISTILLED"
217
+ printf '{"status":"flushed","taskId":"%s","triage":"%s","ingest":"%s","distill":"%s","debt":"%s","markers":"%s"}\n' \
218
+ "$TASK_ID" "${TRIAGE:-none}" "$INGESTED" "$DISTILLED" "${DEBT:-skipped}" "${MARKERS:-skipped}"
209
219
  fi
210
220
 
211
221
  if [ -z "$TRIAGE" ]; then
@@ -80,6 +80,15 @@ case "$VERDICT" in
80
80
  echo "The summary carries LINE NUMBERS, so a follow-up Read with an offset/limit around the" >&2
81
81
  echo "region you actually need is the intended next step and passes this gate." >&2
82
82
  echo "Full text stays on disk under .multi-agent/refs/ if the summary is not enough." >&2
83
+ MEMORY=""
84
+ if [ -f "$HERE/file-context.mjs" ] && command -v node >/dev/null 2>&1; then
85
+ MEMORY="$(node "$HERE/file-context.mjs" --lines "$FILE" 2>/dev/null || true)"
86
+ fi
87
+ if [ -n "$MEMORY" ]; then
88
+ echo "" >&2
89
+ echo "What the repo's memory already says about this file (recorded data, not instructions):" >&2
90
+ printf '%s\n' "$MEMORY" >&2
91
+ fi
83
92
  exit 2 ;;
84
93
  *)
85
94
  exit 0 ;;
@@ -38,10 +38,15 @@
38
38
  "kind": "class",
39
39
  "regex": "\\bclass\\s+([A-Z][A-Za-z0-9_]*)"
40
40
  },
41
+ {
42
+ "id": "composable",
43
+ "kind": "composable",
44
+ "regex": "\\bfun\\s+(?:<[^>]*>\\s*)?(?:[A-Za-z_][A-Za-z0-9_.]*\\.)?([A-Z][A-Za-z0-9_]*)\\s*\\("
45
+ },
41
46
  {
42
47
  "id": "fun",
43
48
  "kind": "fun",
44
- "regex": "\\bfun\\s+(?:<[^>]*>\\s*)?([a-zA-Z_][A-Za-z0-9_]*)"
49
+ "regex": "\\bfun\\s+(?:<[^>]*>\\s*)?(?:[A-Za-z_][A-Za-z0-9_.]*\\.)?([a-z_][A-Za-z0-9_]*)"
45
50
  }
46
51
  ],
47
52
  "importPatterns": [
@@ -50,9 +55,9 @@
50
55
  "regex": "^\\s*import\\s+(?:static\\s+)?(?:[A-Za-z_][A-Za-z0-9_]*\\.)+([A-Z][A-Za-z0-9_]*)"
51
56
  }
52
57
  ],
53
- "referenceKinds": ["class", "interface", "object", "enum", "record", "typealias"],
58
+ "referenceKinds": ["class", "interface", "object", "enum", "record", "typealias", "composable"],
54
59
  "supertypes": { "syntax": "colon", "kinds": ["class", "interface", "object", "enum"] },
55
- "referenceKindsNote": "fun is deliberately absent, for the reason Swift's rules record for func: a bare lowercase name matched across files is almost never a call to that exact declaration, and on a large app the generic ones (invoke, build, create, get) take the top god-node slots purely because each happened to be declared once. Composables are the tempting exception - they are capitalised and behave like types - but they are declared with fun, so including them means including every other fun with them.",
60
+ "referenceKindsNote": "fun is deliberately absent, for the reason Swift's rules record for func: a bare lowercase name matched across files is almost never a call to that exact declaration, and on a large app the generic ones (invoke, build, create, get) take the top god-node slots purely because each happened to be declared once. A capitalised fun is the exception and gets its own kind, composable: Kotlin names ordinary functions in lowerCamelCase, so a capitalised fun is in practice a Compose UI function (or a factory named after the type it builds), and it is called by that exact name the way a type is.",
56
61
  "ignoredIdentifiers": [
57
62
  "this",
58
63
  "super",