@mmerterden/multi-agent-pipeline 12.4.0 → 12.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +169 -0
- package/README.md +1 -1
- package/docs/features.md +20 -0
- package/index.js +7 -1
- package/install/_dev-only-files.mjs +1 -0
- package/package.json +4 -3
- package/pipeline/agents/security-auditor.md +1 -1
- package/pipeline/commands/archive-guard.md +5 -5
- package/pipeline/commands/multi-agent/SKILL.md +2 -0
- package/pipeline/commands/multi-agent/design-check/SKILL.md +287 -0
- package/pipeline/commands/multi-agent/help/SKILL.md +45 -5
- package/pipeline/commands/multi-agent/refactor/SKILL.md +92 -12
- package/pipeline/commands/multi-agent/review/SKILL.md +1 -1
- package/pipeline/commands/multi-agent/sync/SKILL.md +119 -12
- package/pipeline/commands/multi-agent/test/SKILL.md +1 -1
- package/pipeline/commands/sim-test.md +5 -5
- package/pipeline/lib/credential-store.sh +32 -0
- package/pipeline/lib/extract-conventions.sh +1 -0
- package/pipeline/lib/repo-cache.sh +1 -0
- package/pipeline/lib/shadow-git.sh +8 -0
- package/pipeline/multi-agent-refs/cross-cli-contract.md +3 -3
- package/pipeline/multi-agent-refs/phases/phase-0-init.md +9 -1
- package/pipeline/multi-agent-refs/phases/phase-3-dev.md +2 -2
- package/pipeline/multi-agent-refs/phases/phase-4-review.md +3 -2
- package/pipeline/multi-agent-refs/phases/phase-5-test.md +14 -4
- package/pipeline/multi-agent-refs/phases.md +2 -0
- package/pipeline/multi-agent-refs/prompt-assembly.md +31 -0
- package/pipeline/preferences-template.json +18 -1
- package/pipeline/schemas/agent-state.schema.json +90 -0
- package/pipeline/schemas/design-check-config.schema.json +162 -0
- package/pipeline/schemas/learnings-ledger.schema.json +4 -0
- package/pipeline/schemas/migrations/state-2.0.0-to-2.1.0.mjs +30 -12
- package/pipeline/schemas/prefs.schema.json +161 -5
- package/pipeline/schemas/token-budget.json +2 -2
- package/pipeline/schemas/triage-corpus.schema.json +5 -1
- package/pipeline/scripts/README.md +7 -5
- package/pipeline/scripts/classify-plan-safety.mjs +8 -3
- package/pipeline/scripts/cost-budget-check.mjs +9 -5
- package/pipeline/scripts/eval-mine-corpus.mjs +19 -5
- package/pipeline/scripts/fixtures/install-layout.tsv +7 -7
- package/pipeline/scripts/gc-worktrees.sh +23 -1
- package/pipeline/scripts/learning-curve.mjs +167 -0
- package/pipeline/scripts/learnings-ledger.mjs +27 -2
- package/pipeline/scripts/lint-mcp-refs.mjs +207 -0
- package/pipeline/scripts/memory-load.sh +5 -1
- package/pipeline/scripts/render-work-summary.sh +4 -1
- package/pipeline/scripts/repo-map.mjs +1 -1
- package/pipeline/scripts/smoke-command-inventory.sh +81 -0
- package/pipeline/scripts/smoke-commands-skills-parity.sh +1 -1
- package/pipeline/scripts/smoke-compliance-skills.sh +4 -4
- package/pipeline/scripts/smoke-cross-cli-behavior.sh +12 -2
- package/pipeline/scripts/smoke-generate-issue.sh +6 -5
- package/pipeline/scripts/smoke-learning-curve.sh +61 -0
- package/pipeline/scripts/smoke-per-repo-memory.sh +2 -2
- package/pipeline/scripts/smoke-review-readiness.sh +3 -2
- package/pipeline/scripts/smoke-schema-validation.sh +19 -5
- package/pipeline/scripts/smoke-shadow-git.sh +4 -2
- package/pipeline/scripts/triage-memory.mjs +66 -10
- package/pipeline/scripts/uninstall.mjs +1 -1
- package/pipeline/skills/.skill-manifest.json +24 -8
- package/pipeline/skills/.skills-index.json +39 -3
- package/pipeline/skills/shared/README.md +10 -6
- package/pipeline/skills/shared/core/apple-archive-compliance/SKILL.md +10 -8
- package/pipeline/skills/shared/core/multi-agent-design-check/SKILL.md +248 -0
- package/pipeline/skills/shared/core/multi-agent-help/SKILL.md +22 -0
- package/pipeline/skills/shared/core/multi-agent-refactor/SKILL.md +67 -11
- package/pipeline/skills/shared/core/multi-agent-review/SKILL.md +1 -1
- package/pipeline/skills/shared/core/multi-agent-sync/SKILL.md +51 -9
- package/pipeline/skills/shared/core/multi-agent-test/SKILL.md +1 -1
- package/pipeline/skills/skills-index.md +8 -4
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
|
|
34
34
|
import { readFileSync, existsSync } from "fs";
|
|
35
35
|
import { dirname, join } from "path";
|
|
36
|
+
import { homedir } from "os";
|
|
36
37
|
import { fileURLToPath } from "url";
|
|
37
38
|
import { costUsd } from "./_cost.mjs";
|
|
38
39
|
|
|
@@ -103,10 +104,11 @@ function resolveTracker() {
|
|
|
103
104
|
const id = flags["task-id"];
|
|
104
105
|
if (!id) die("either --tracker or --task-id is required");
|
|
105
106
|
const bare = String(id).replace(/^#/, "");
|
|
107
|
+
const logRoot = join(homedir(), ".claude", "logs", "multi-agent");
|
|
106
108
|
for (const c of [
|
|
107
|
-
join(
|
|
108
|
-
join(
|
|
109
|
-
join(
|
|
109
|
+
join(logRoot, String(id), "tracker-state.json"),
|
|
110
|
+
join(logRoot, bare, "tracker-state.json"),
|
|
111
|
+
join(logRoot, `task-${bare}`, "tracker-state.json"),
|
|
110
112
|
]) {
|
|
111
113
|
if (existsSync(c)) return c;
|
|
112
114
|
}
|
|
@@ -134,15 +136,17 @@ if (!rate) die(`no price for model "${cfg.pricingModel}" in cost-table.json`);
|
|
|
134
136
|
|
|
135
137
|
let tokensIn = 0;
|
|
136
138
|
let tokensOut = 0;
|
|
139
|
+
let tokensCached = 0;
|
|
137
140
|
for (const p of Object.values(tracker.phases || {})) {
|
|
138
141
|
tokensIn += Number(p?.tokens_in || 0);
|
|
139
142
|
tokensOut += Number(p?.tokens_out || 0);
|
|
143
|
+
tokensCached += Number(p?.tokens_cached || 0);
|
|
140
144
|
}
|
|
141
145
|
|
|
142
|
-
const usd = costUsd(rate, tokensIn, tokensOut);
|
|
146
|
+
const usd = costUsd(rate, tokensIn, tokensOut, tokensCached);
|
|
143
147
|
const pctOfMax = (usd / cfg.maxUsd) * 100;
|
|
144
148
|
const usdStr = `$${usd.toFixed(2)}`;
|
|
145
|
-
const totalTok = tokensIn + tokensOut;
|
|
149
|
+
const totalTok = tokensIn + tokensOut + tokensCached;
|
|
146
150
|
|
|
147
151
|
const base =
|
|
148
152
|
`cost-budget: ${usdStr} of $${cfg.maxUsd.toFixed(2)} ` +
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// eval-mine-corpus.mjs - grow the triage eval set from real production runs.
|
|
3
3
|
//
|
|
4
|
-
// The pipeline already records every Phase 4 triage decision to
|
|
5
|
-
// `~/.claude/
|
|
4
|
+
// The pipeline already records every Phase 4 triage decision to the per-repo
|
|
5
|
+
// `~/.claude/memory/multi-agent/<repo-slug>/triage-corpus.jsonl` (one line per finding:
|
|
6
6
|
// severity/file/line/issue/fix/reviewer + the classification the run gave it,
|
|
7
7
|
// plus an optional human `reason`/`reviewer`). Hand-authoring eval fixtures is
|
|
8
8
|
// the bottleneck; this closes the loop by turning those real decisions into
|
|
@@ -38,19 +38,33 @@ const candidatesDir = join(root, "pipeline", "eval", "triage-candidates");
|
|
|
38
38
|
const validator = join(root, "pipeline", "scripts", "validate-triage.mjs");
|
|
39
39
|
|
|
40
40
|
const args = process.argv.slice(2);
|
|
41
|
-
const opts = { corpus: null, limit: Infinity, dryRun: false };
|
|
41
|
+
const opts = { corpus: null, limit: Infinity, dryRun: false, repoSlug: null };
|
|
42
42
|
for (let i = 0; i < args.length; i++) {
|
|
43
43
|
if (args[i] === "--corpus") opts.corpus = args[++i];
|
|
44
|
+
else if (args[i] === "--repo-slug") opts.repoSlug = args[++i];
|
|
44
45
|
else if (args[i] === "--limit") opts.limit = Number(args[++i]) || Infinity;
|
|
45
46
|
else if (args[i] === "--dry-run") opts.dryRun = true;
|
|
46
47
|
else if (args[i] === "-h" || args[i] === "--help") {
|
|
47
|
-
console.log("Usage: eval-mine-corpus.mjs [--corpus <path>] [--limit N] [--dry-run]");
|
|
48
|
+
console.log("Usage: eval-mine-corpus.mjs [--corpus <path>] [--repo-slug <slug>] [--limit N] [--dry-run]");
|
|
48
49
|
process.exit(0);
|
|
49
50
|
}
|
|
50
51
|
}
|
|
51
52
|
|
|
53
|
+
// Resolve the corpus path the SAME way triage-memory.mjs writes it:
|
|
54
|
+
// ~/.claude/memory/multi-agent/<repo-slug>/triage-corpus.jsonl (per repo).
|
|
55
|
+
// (The old default read ~/.claude/logs/.../triage-corpus.jsonl - a flat path
|
|
56
|
+
// nothing writes to - so without --corpus it always found nothing.)
|
|
57
|
+
function resolveRepoSlug() {
|
|
58
|
+
if (opts.repoSlug) return String(opts.repoSlug);
|
|
59
|
+
if (process.env.MULTI_AGENT_REPO_SLUG) return process.env.MULTI_AGENT_REPO_SLUG;
|
|
60
|
+
const r = spawnSync("git", ["rev-parse", "--show-toplevel"], { encoding: "utf-8" });
|
|
61
|
+
if (r.status === 0 && r.stdout.trim()) return r.stdout.trim().split("/").pop();
|
|
62
|
+
return "default";
|
|
63
|
+
}
|
|
64
|
+
|
|
52
65
|
const corpusPath =
|
|
53
|
-
opts.corpus ||
|
|
66
|
+
opts.corpus ||
|
|
67
|
+
join(homedir(), ".claude", "memory", "multi-agent", resolveRepoSlug(), "triage-corpus.jsonl");
|
|
54
68
|
|
|
55
69
|
if (!existsSync(corpusPath)) {
|
|
56
70
|
console.log(`eval-mine-corpus: no corpus at ${corpusPath} - nothing to mine.`);
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
.claude/.pipeline-version 1
|
|
2
2
|
.claude/CLAUDE.md 1
|
|
3
3
|
.claude/agents 8
|
|
4
|
-
.claude/commands
|
|
4
|
+
.claude/commands 48
|
|
5
5
|
.claude/lib 27
|
|
6
6
|
.claude/multi-agent-preferences.json 1
|
|
7
|
-
.claude/multi-agent-refs
|
|
7
|
+
.claude/multi-agent-refs 52
|
|
8
8
|
.claude/rules 12
|
|
9
|
-
.claude/schemas
|
|
10
|
-
.claude/scripts
|
|
9
|
+
.claude/schemas 25
|
|
10
|
+
.claude/scripts 201
|
|
11
11
|
.claude/settings.json 1
|
|
12
12
|
.claude/skills 428
|
|
13
13
|
.copilot/.pipeline-version 1
|
|
14
14
|
.copilot/agents 8
|
|
15
15
|
.copilot/copilot-instructions.md 1
|
|
16
16
|
.copilot/lib 27
|
|
17
|
-
.copilot/schemas
|
|
18
|
-
.copilot/scripts
|
|
19
|
-
.copilot/skills
|
|
17
|
+
.copilot/schemas 25
|
|
18
|
+
.copilot/scripts 201
|
|
19
|
+
.copilot/skills 471
|
|
@@ -64,6 +64,13 @@ if ! git -C "$REPO" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
|
|
64
64
|
exit 0
|
|
65
65
|
fi
|
|
66
66
|
REPO="$(cd "$REPO" && pwd -P)"
|
|
67
|
+
# Anchor on the MAIN worktree (first porcelain entry) so a run started from a
|
|
68
|
+
# subdirectory OR from inside a linked worktree still targets
|
|
69
|
+
# <main-repo>/.worktrees/ rather than a bogus <subdir>/.worktrees/ (which
|
|
70
|
+
# silently found nothing and left residue behind).
|
|
71
|
+
MAIN_WT="$(git -C "$REPO" worktree list --porcelain 2>/dev/null | sed -n 's/^worktree //p' | head -1)"
|
|
72
|
+
MAIN_WT="$(cd "$MAIN_WT" 2>/dev/null && pwd -P)" || MAIN_WT=""
|
|
73
|
+
[ -n "$MAIN_WT" ] && REPO="$MAIN_WT"
|
|
67
74
|
WT_ROOT="$REPO/.worktrees"
|
|
68
75
|
|
|
69
76
|
changed=0
|
|
@@ -76,9 +83,24 @@ pruned=$(git -C "$REPO" worktree prune -v 2>&1 | grep -c . || true)
|
|
|
76
83
|
# 2. Orphan dirs: present under .worktrees/ but not registered as worktrees.
|
|
77
84
|
orphans=()
|
|
78
85
|
if [ -d "$WT_ROOT" ]; then
|
|
79
|
-
|
|
86
|
+
# Resolve every registered worktree path to its physical form so the
|
|
87
|
+
# orphan test below compares like-for-like against `pwd -P`-resolved dirs.
|
|
88
|
+
# A raw (unresolved) registered path with a symlinked component would fail
|
|
89
|
+
# the match and get a HEALTHY worktree misclassified as an orphan and
|
|
90
|
+
# deleted under --yes; resolving both sides removes that risk (this mirrors
|
|
91
|
+
# purge.sh, which resolves both sides).
|
|
92
|
+
registered=""
|
|
93
|
+
while IFS= read -r regpath; do
|
|
94
|
+
[ -n "$regpath" ] || continue
|
|
95
|
+
regphys="$(cd "$regpath" 2>/dev/null && pwd -P)" || regphys=""
|
|
96
|
+
[ -n "$regphys" ] && registered="$registered$regphys
|
|
97
|
+
"
|
|
98
|
+
done <<EOF
|
|
99
|
+
$(git -C "$REPO" worktree list --porcelain 2>/dev/null | sed -n 's/^worktree //p')
|
|
100
|
+
EOF
|
|
80
101
|
for d in "$WT_ROOT"/*/; do
|
|
81
102
|
[ -d "$d" ] || continue
|
|
103
|
+
[ -L "${d%/}" ] && continue # never follow a symlinked entry out of the tree
|
|
82
104
|
dir="$(cd "$d" && pwd -P)"
|
|
83
105
|
# Path guard: only ever consider dirs strictly inside <repo>/.worktrees/.
|
|
84
106
|
case "$dir" in "$WT_ROOT"/*) ;; *) continue ;; esac
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// learning-curve.mjs - time-bucketed TREND over metrics.jsonl.
|
|
3
|
+
//
|
|
4
|
+
// aggregate-metrics.mjs reports lifetime totals; this reports the CURVE: how
|
|
5
|
+
// the pipeline's per-task quality/cost move over time as it accumulates
|
|
6
|
+
// learnings on a repo. Without a trend you cannot tell whether "it gets better
|
|
7
|
+
// over time" is real - this is the measurement layer for the learning loop.
|
|
8
|
+
//
|
|
9
|
+
// Buckets tasks by the timestamp of their first event, then per bucket reports:
|
|
10
|
+
// tasks, first-pass-clean rate, avg review cycles, rework/task,
|
|
11
|
+
// tokens/task, cache-reuse ratio. Prints a first->last trend for the
|
|
12
|
+
// headline KPIs. Read-only, zero deps, no API/model calls.
|
|
13
|
+
//
|
|
14
|
+
// Usage:
|
|
15
|
+
// learning-curve.mjs [--bucket=<days>] [--since=ISO] [--json|--markdown]
|
|
16
|
+
// Env: METRICS_FILE overrides the default ~/.claude/logs/multi-agent/metrics.jsonl
|
|
17
|
+
//
|
|
18
|
+
// Exit: 0 always (advisory; empty/missing metrics -> friendly note).
|
|
19
|
+
|
|
20
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
21
|
+
import { homedir } from "node:os";
|
|
22
|
+
import { join } from "node:path";
|
|
23
|
+
|
|
24
|
+
const args = process.argv.slice(2);
|
|
25
|
+
const opts = { json: false, markdown: false, since: null, bucketDays: 7 };
|
|
26
|
+
for (const a of args) {
|
|
27
|
+
if (a === "--json") opts.json = true;
|
|
28
|
+
else if (a === "--markdown" || a === "--md") opts.markdown = true;
|
|
29
|
+
else if (a.startsWith("--since=")) opts.since = a.slice(8);
|
|
30
|
+
else if (a.startsWith("--bucket=")) opts.bucketDays = Math.max(1, parseInt(a.slice(9), 10) || 7);
|
|
31
|
+
else if (a === "--help" || a === "-h") {
|
|
32
|
+
console.log("Usage: learning-curve.mjs [--bucket=<days>] [--since=ISO] [--json|--markdown]");
|
|
33
|
+
process.exit(0);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const METRICS_FILE =
|
|
38
|
+
process.env.METRICS_FILE || join(homedir(), ".claude", "logs", "multi-agent", "metrics.jsonl");
|
|
39
|
+
|
|
40
|
+
function emitEmpty(reason) {
|
|
41
|
+
if (opts.json) console.log(JSON.stringify({ ok: false, reason, buckets: [] }));
|
|
42
|
+
else console.error(`learning-curve: ${reason}`);
|
|
43
|
+
process.exit(0);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
if (!existsSync(METRICS_FILE)) emitEmpty(`no metrics file at ${METRICS_FILE}`);
|
|
47
|
+
|
|
48
|
+
const events = [];
|
|
49
|
+
for (const line of readFileSync(METRICS_FILE, "utf-8").split("\n")) {
|
|
50
|
+
if (!line.trim()) continue;
|
|
51
|
+
try {
|
|
52
|
+
const e = JSON.parse(line);
|
|
53
|
+
if (opts.since && e.ts && e.ts < opts.since) continue;
|
|
54
|
+
events.push(e);
|
|
55
|
+
} catch {
|
|
56
|
+
/* skip corrupted line */
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if (events.length === 0) emitEmpty("no metrics events in range");
|
|
60
|
+
|
|
61
|
+
// Fold events into per-task aggregates.
|
|
62
|
+
const num = (v) => (typeof v === "number" ? v : Number(v)) || 0;
|
|
63
|
+
const tasks = new Map();
|
|
64
|
+
for (const e of events) {
|
|
65
|
+
const id = e.task_id;
|
|
66
|
+
if (!id) continue;
|
|
67
|
+
let t = tasks.get(id);
|
|
68
|
+
if (!t) {
|
|
69
|
+
t = { firstTs: e.ts, reviewCycles: null, rework: 0, tokensIn: 0, tokensOut: 0, tokensCached: 0 };
|
|
70
|
+
tasks.set(id, t);
|
|
71
|
+
}
|
|
72
|
+
if (e.ts && (!t.firstTs || e.ts < t.firstTs)) t.firstTs = e.ts;
|
|
73
|
+
const d = e.details || {};
|
|
74
|
+
if (e.event === "review.completed" && d.review_cycles != null) t.reviewCycles = num(d.review_cycles);
|
|
75
|
+
if (e.event === "rework.started") t.rework += 1;
|
|
76
|
+
t.tokensIn += num(d.tokens_in);
|
|
77
|
+
t.tokensOut += num(d.tokens_out);
|
|
78
|
+
t.tokensCached += num(d.tokens_cached);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Bucket tasks by first-seen day / bucketDays window (UTC).
|
|
82
|
+
const DAY = 86400000;
|
|
83
|
+
const bucketOf = (ts) => {
|
|
84
|
+
const ms = Date.parse(ts);
|
|
85
|
+
if (Number.isNaN(ms)) return null;
|
|
86
|
+
const dayIndex = Math.floor(ms / DAY);
|
|
87
|
+
const b = Math.floor(dayIndex / opts.bucketDays) * opts.bucketDays;
|
|
88
|
+
return new Date(b * DAY).toISOString().slice(0, 10);
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
const byBucket = new Map();
|
|
92
|
+
for (const t of tasks.values()) {
|
|
93
|
+
const key = bucketOf(t.firstTs);
|
|
94
|
+
if (!key) continue;
|
|
95
|
+
if (!byBucket.has(key)) byBucket.set(key, []);
|
|
96
|
+
byBucket.get(key).push(t);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const round = (n, p = 2) => Math.round(n * 10 ** p) / 10 ** p;
|
|
100
|
+
const buckets = [...byBucket.keys()].sort().map((start) => {
|
|
101
|
+
const list = byBucket.get(start);
|
|
102
|
+
const n = list.length;
|
|
103
|
+
const withReview = list.filter((t) => t.reviewCycles != null);
|
|
104
|
+
const firstPassClean = withReview.filter((t) => t.reviewCycles <= 1).length;
|
|
105
|
+
const totIn = list.reduce((s, t) => s + t.tokensIn, 0);
|
|
106
|
+
const totOut = list.reduce((s, t) => s + t.tokensOut, 0);
|
|
107
|
+
const totCached = list.reduce((s, t) => s + t.tokensCached, 0);
|
|
108
|
+
return {
|
|
109
|
+
bucketStart: start,
|
|
110
|
+
tasks: n,
|
|
111
|
+
firstPassCleanRate: withReview.length ? round(firstPassClean / withReview.length) : null,
|
|
112
|
+
avgReviewCycles: withReview.length
|
|
113
|
+
? round(withReview.reduce((s, t) => s + t.reviewCycles, 0) / withReview.length)
|
|
114
|
+
: null,
|
|
115
|
+
reworkPerTask: round(list.reduce((s, t) => s + t.rework, 0) / n),
|
|
116
|
+
tokensPerTask: Math.round((totIn + totOut) / n),
|
|
117
|
+
cacheRatio: totIn ? round(totCached / totIn) : null,
|
|
118
|
+
};
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
// first->last trend for the headline KPIs (only when >=2 buckets).
|
|
122
|
+
let trend = null;
|
|
123
|
+
if (buckets.length >= 2) {
|
|
124
|
+
const a = buckets[0];
|
|
125
|
+
const z = buckets[buckets.length - 1];
|
|
126
|
+
const delta = (x, y) => (x == null || y == null ? null : round(y - x));
|
|
127
|
+
trend = {
|
|
128
|
+
from: a.bucketStart,
|
|
129
|
+
to: z.bucketStart,
|
|
130
|
+
firstPassCleanRate: delta(a.firstPassCleanRate, z.firstPassCleanRate),
|
|
131
|
+
avgReviewCycles: delta(a.avgReviewCycles, z.avgReviewCycles),
|
|
132
|
+
tokensPerTask: a.tokensPerTask && z.tokensPerTask ? z.tokensPerTask - a.tokensPerTask : null,
|
|
133
|
+
cacheRatio: delta(a.cacheRatio, z.cacheRatio),
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const out = { ok: true, bucketDays: opts.bucketDays, buckets, trend };
|
|
138
|
+
|
|
139
|
+
if (opts.json) {
|
|
140
|
+
console.log(JSON.stringify(out, null, 2));
|
|
141
|
+
process.exit(0);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Markdown / text (default)
|
|
145
|
+
const arrow = (v, goodIsUp) => {
|
|
146
|
+
if (v == null || v === 0) return "-";
|
|
147
|
+
const up = v > 0;
|
|
148
|
+
const good = up === goodIsUp;
|
|
149
|
+
return `${up ? "+" : ""}${v} ${good ? "(better)" : "(worse)"}`;
|
|
150
|
+
};
|
|
151
|
+
console.log(`# Learning curve (${opts.bucketDays}-day buckets)\n`);
|
|
152
|
+
console.log("| Bucket | Tasks | First-pass clean | Avg review cycles | Rework/task | Tokens/task | Cache ratio |");
|
|
153
|
+
console.log("|---|---|---|---|---|---|---|");
|
|
154
|
+
for (const b of buckets) {
|
|
155
|
+
console.log(
|
|
156
|
+
`| ${b.bucketStart} | ${b.tasks} | ${b.firstPassCleanRate ?? "-"} | ${b.avgReviewCycles ?? "-"} | ${b.reworkPerTask} | ${b.tokensPerTask} | ${b.cacheRatio ?? "-"} |`,
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
if (trend) {
|
|
160
|
+
console.log(`\n**Trend ${trend.from} -> ${trend.to}:**`);
|
|
161
|
+
console.log(`- First-pass clean: ${arrow(trend.firstPassCleanRate, true)}`);
|
|
162
|
+
console.log(`- Avg review cycles: ${arrow(trend.avgReviewCycles, false)}`);
|
|
163
|
+
console.log(`- Tokens/task: ${arrow(trend.tokensPerTask, false)}`);
|
|
164
|
+
console.log(`- Cache ratio: ${arrow(trend.cacheRatio, true)}`);
|
|
165
|
+
} else {
|
|
166
|
+
console.log("\n(need >= 2 buckets for a trend; keep running the pipeline)");
|
|
167
|
+
}
|
|
@@ -74,9 +74,27 @@ function die(msg, code = 1) {
|
|
|
74
74
|
process.exit(code);
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
// Resolves to the MAIN repository name even when called from inside a pipeline
|
|
78
|
+
// worktree. `--show-toplevel` returns the worktree root, whose basename is the
|
|
79
|
+
// task id, which would shard the per-repo ledger one directory per task and make
|
|
80
|
+
// the learning replay always miss. `--git-common-dir` points at the main
|
|
81
|
+
// repository's .git for every linked worktree.
|
|
77
82
|
function repoSlug() {
|
|
78
83
|
if (flags["repo-slug"]) return String(flags["repo-slug"]);
|
|
79
84
|
if (process.env.MULTI_AGENT_REPO_SLUG) return process.env.MULTI_AGENT_REPO_SLUG;
|
|
85
|
+
try {
|
|
86
|
+
const commonDir = execFileSync(
|
|
87
|
+
"git",
|
|
88
|
+
["rev-parse", "--path-format=absolute", "--git-common-dir"],
|
|
89
|
+
{ encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }
|
|
90
|
+
).trim();
|
|
91
|
+
if (commonDir) {
|
|
92
|
+
const base = basename(commonDir);
|
|
93
|
+
const root = base === ".git" ? dirname(commonDir) : commonDir.replace(/\.git$/, "");
|
|
94
|
+
const slug = basename(root);
|
|
95
|
+
if (slug) return slug;
|
|
96
|
+
}
|
|
97
|
+
} catch { /* not a git repo, or git too old for --path-format */ }
|
|
80
98
|
try {
|
|
81
99
|
const top = execFileSync("git", ["rev-parse", "--show-toplevel"], {
|
|
82
100
|
encoding: "utf8",
|
|
@@ -120,7 +138,7 @@ function writeRow(slug, row) {
|
|
|
120
138
|
|
|
121
139
|
/** Append one entry unless an entry with the same kind + normalized statement
|
|
122
140
|
* already exists (idempotent). Returns true when written. */
|
|
123
|
-
function addEntry(slug, { kind, statement, scope, task, confidence }, existing) {
|
|
141
|
+
function addEntry(slug, { kind, statement, scope, task, confidence, diagnosis }, existing) {
|
|
124
142
|
const stmt = String(statement || "").trim();
|
|
125
143
|
if (!stmt) return false;
|
|
126
144
|
const seenKey = `${kind}::${normStatement(stmt)}`;
|
|
@@ -135,6 +153,11 @@ function addEntry(slug, { kind, statement, scope, task, confidence }, existing)
|
|
|
135
153
|
source_task: task || null,
|
|
136
154
|
confidence: CONFIDENCES.has(confidence) ? confidence : "medium",
|
|
137
155
|
};
|
|
156
|
+
// Optional causal diagnosis (Reflexion P5): the "why it failed" behind a
|
|
157
|
+
// lesson. Verbal diagnosis is what turns a detected error into a prevented
|
|
158
|
+
// one; a bare outcome does not. Stored only when supplied.
|
|
159
|
+
const diag = String(diagnosis || "").trim();
|
|
160
|
+
if (diag) row.diagnosis = diag;
|
|
138
161
|
writeRow(slug, row);
|
|
139
162
|
existing.add(seenKey);
|
|
140
163
|
return true;
|
|
@@ -152,6 +175,7 @@ function cmdAdd() {
|
|
|
152
175
|
scope: flags.scope && flags.scope !== true ? String(flags.scope) : "global",
|
|
153
176
|
task: flags.task && flags.task !== true ? String(flags.task) : null,
|
|
154
177
|
confidence: flags.confidence && flags.confidence !== true ? String(flags.confidence) : "medium",
|
|
178
|
+
diagnosis: flags.diagnosis && flags.diagnosis !== true ? String(flags.diagnosis) : null,
|
|
155
179
|
}, existing);
|
|
156
180
|
process.stdout.write(JSON.stringify({ ok: true, slug, added: wrote ? 1 : 0, deduped: wrote ? 0 : 1 }) + "\n");
|
|
157
181
|
process.exit(wrote ? 0 : 2);
|
|
@@ -247,7 +271,8 @@ function cmdBrief() {
|
|
|
247
271
|
lines.push("", `## ${labels[kind]}`);
|
|
248
272
|
for (const r of items) {
|
|
249
273
|
const scope = r.scope && r.scope !== "global" ? ` [${r.scope}]` : "";
|
|
250
|
-
|
|
274
|
+
const why = r.diagnosis ? ` (why: ${r.diagnosis})` : "";
|
|
275
|
+
lines.push(`- ${r.statement}${scope}${why}`);
|
|
251
276
|
}
|
|
252
277
|
}
|
|
253
278
|
lines.push("</repo-learnings>");
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// lint-mcp-refs.mjs - every mcp__<server>__<tool> reference must be callable.
|
|
3
|
+
//
|
|
4
|
+
// A host composes an MCP tool name as mcp__<registered-server-name>__<tool>.
|
|
5
|
+
// The pipeline had 100 references to `mcp__dev_toolkit__*` while the server is
|
|
6
|
+
// registered as `dev-toolkit`, so the real names are `mcp__dev-toolkit__*`.
|
|
7
|
+
// Nothing caught it: prose is not executed, and a wrong name inside an
|
|
8
|
+
// `allowed-tools:` allowlist silently denies the tool instead of erroring.
|
|
9
|
+
//
|
|
10
|
+
// Checks (errors - fail the gate):
|
|
11
|
+
// 1. the server segment must be a known registered server name
|
|
12
|
+
// 2. the known-bad snake_case spellings are called out with their fix
|
|
13
|
+
//
|
|
14
|
+
// Optional deep check (advisory - never fails):
|
|
15
|
+
// 3. when the companion dev-toolkit repo resolves locally (via
|
|
16
|
+
// prefs.global.devToolkit.localPath, or the mcpServers registration in
|
|
17
|
+
// ~/.claude.json / ~/.claude/settings.json), handshake it over stdio and
|
|
18
|
+
// confirm every referenced dev-toolkit tool actually exists in tools/list.
|
|
19
|
+
// CI has no local checkout, so an unresolvable repo is a skip, not a fail.
|
|
20
|
+
//
|
|
21
|
+
// Zero-dependency. Runs inside `npm test` and ci-lite.
|
|
22
|
+
//
|
|
23
|
+
// Usage: node pipeline/scripts/lint-mcp-refs.mjs [--strict-tools]
|
|
24
|
+
// Exit: 0 clean, 1 unknown server segment (or a missing tool with --strict-tools).
|
|
25
|
+
|
|
26
|
+
import { readFileSync, existsSync, readdirSync, statSync } from "node:fs";
|
|
27
|
+
import { homedir } from "node:os";
|
|
28
|
+
import { dirname, join, resolve } from "node:path";
|
|
29
|
+
import { fileURLToPath } from "node:url";
|
|
30
|
+
import { spawnSync } from "node:child_process";
|
|
31
|
+
|
|
32
|
+
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
33
|
+
const strictTools = process.argv.includes("--strict-tools");
|
|
34
|
+
|
|
35
|
+
// Servers the pipeline is allowed to reference. The key is the registered
|
|
36
|
+
// server name as it appears in an mcpServers config, which is what the host
|
|
37
|
+
// interpolates into the tool name.
|
|
38
|
+
const KNOWN_SERVERS = {
|
|
39
|
+
"dev-toolkit": "companion device/browser toolkit (@mmerterden/dev-toolkit-mcp)",
|
|
40
|
+
claude_ai_Figma: "Figma MCP (analysis phase only)",
|
|
41
|
+
XcodeBuildMCP: "optional third-party Xcode build server",
|
|
42
|
+
};
|
|
43
|
+
// Spellings that look right but cannot resolve to a registered server.
|
|
44
|
+
const KNOWN_BAD = { dev_toolkit: "dev-toolkit", "claude-ai-Figma": "claude_ai_Figma" };
|
|
45
|
+
|
|
46
|
+
const SCAN_DIRS = ["pipeline/commands", "pipeline/skills", "pipeline/multi-agent-refs"];
|
|
47
|
+
const SCAN_EXT = /\.(md|sh|mjs|json)$/;
|
|
48
|
+
const PRUNE = new Set(["node_modules", ".git", ".worktrees", "Pods", ".build", "DerivedData", ".next"]);
|
|
49
|
+
|
|
50
|
+
function walk(dir, out = []) {
|
|
51
|
+
let entries;
|
|
52
|
+
try {
|
|
53
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
54
|
+
} catch {
|
|
55
|
+
return out;
|
|
56
|
+
}
|
|
57
|
+
for (const e of entries) {
|
|
58
|
+
if (PRUNE.has(e.name)) continue;
|
|
59
|
+
const p = join(dir, e.name);
|
|
60
|
+
if (e.isDirectory()) walk(p, out);
|
|
61
|
+
else if (SCAN_EXT.test(e.name)) out.push(p);
|
|
62
|
+
}
|
|
63
|
+
return out;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const files = SCAN_DIRS.flatMap((d) => walk(join(repoRoot, d)));
|
|
67
|
+
const REF = /mcp__([A-Za-z0-9_-]+?)__([a-z0-9_]+)/g;
|
|
68
|
+
|
|
69
|
+
const errors = [];
|
|
70
|
+
const refs = new Map(); // server -> Map(tool -> Set(file:line))
|
|
71
|
+
|
|
72
|
+
for (const file of files) {
|
|
73
|
+
const rel = file.slice(repoRoot.length + 1);
|
|
74
|
+
const lines = readFileSync(file, "utf8").split("\n");
|
|
75
|
+
lines.forEach((line, i) => {
|
|
76
|
+
for (const m of line.matchAll(REF)) {
|
|
77
|
+
const [, server, tool] = m;
|
|
78
|
+
// Prose writes families as mcp__dev-toolkit__ios_* - a wildcard, not a tool.
|
|
79
|
+
const next = line[m.index + m[0].length];
|
|
80
|
+
if (next === "*" || tool.endsWith("_")) continue;
|
|
81
|
+
const where = `${rel}:${i + 1}`;
|
|
82
|
+
if (!refs.has(server)) refs.set(server, new Map());
|
|
83
|
+
const tools = refs.get(server);
|
|
84
|
+
if (!tools.has(tool)) tools.set(tool, new Set());
|
|
85
|
+
tools.get(tool).add(where);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
for (const [server, tools] of refs) {
|
|
91
|
+
if (server in KNOWN_SERVERS) continue;
|
|
92
|
+
const fix = KNOWN_BAD[server];
|
|
93
|
+
const sample = [...tools.values()][0].values().next().value;
|
|
94
|
+
errors.push(
|
|
95
|
+
fix
|
|
96
|
+
? `unknown MCP server segment "${server}" (${tools.size} tool refs, e.g. ${sample}) - the registered server is "${fix}", so the prefix must be mcp__${fix}__`
|
|
97
|
+
: `unknown MCP server segment "${server}" (${tools.size} tool refs, e.g. ${sample}) - add it to KNOWN_SERVERS in lint-mcp-refs.mjs if this server is expected, or fix the prefix to match the registered server name`,
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// ---- optional deep check: do the referenced dev-toolkit tools exist? --------
|
|
102
|
+
|
|
103
|
+
function resolveToolkitRepo() {
|
|
104
|
+
const prefsPath = join(homedir(), ".claude", "multi-agent-preferences.json");
|
|
105
|
+
if (existsSync(prefsPath)) {
|
|
106
|
+
try {
|
|
107
|
+
const dt = JSON.parse(readFileSync(prefsPath, "utf8"))?.global?.devToolkit;
|
|
108
|
+
if (dt && dt.enabled === false) return null;
|
|
109
|
+
if (dt?.localPath) {
|
|
110
|
+
const p = dt.localPath.replace(/^~|\$HOME/, homedir());
|
|
111
|
+
if (existsSync(join(p, "package.json"))) return p;
|
|
112
|
+
}
|
|
113
|
+
} catch {
|
|
114
|
+
/* fall through to registration lookup */
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
const candidates = [];
|
|
118
|
+
const collect = (obj) => {
|
|
119
|
+
for (const [name, cfg] of Object.entries(obj || {})) {
|
|
120
|
+
const arg = Array.isArray(cfg?.args) ? cfg.args.find((a) => /\.(mjs|js)$/.test(a)) : null;
|
|
121
|
+
if (name.includes("dev") && arg) candidates.push(dirname(resolve(arg)));
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
for (const cfgPath of [join(homedir(), ".claude.json"), join(homedir(), ".claude", "settings.json")]) {
|
|
125
|
+
if (!existsSync(cfgPath)) continue;
|
|
126
|
+
try {
|
|
127
|
+
const json = JSON.parse(readFileSync(cfgPath, "utf8"));
|
|
128
|
+
collect(json.mcpServers);
|
|
129
|
+
for (const proj of Object.values(json.projects || {})) collect(proj.mcpServers);
|
|
130
|
+
} catch {
|
|
131
|
+
/* ignore unreadable config */
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
for (const c of candidates) {
|
|
135
|
+
try {
|
|
136
|
+
const pkg = JSON.parse(readFileSync(join(c, "package.json"), "utf8"));
|
|
137
|
+
if (pkg.dependencies?.["@modelcontextprotocol/sdk"] && statSync(join(c, ".git"))) return c;
|
|
138
|
+
} catch {
|
|
139
|
+
/* not the toolkit */
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function listTools(repo) {
|
|
146
|
+
const frames =
|
|
147
|
+
[
|
|
148
|
+
'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"lint-mcp-refs","version":"1"}}}',
|
|
149
|
+
'{"jsonrpc":"2.0","method":"notifications/initialized"}',
|
|
150
|
+
'{"jsonrpc":"2.0","id":2,"method":"tools/list"}',
|
|
151
|
+
].join("\n") + "\n";
|
|
152
|
+
const r = spawnSync(process.execPath, ["index.js"], {
|
|
153
|
+
cwd: repo,
|
|
154
|
+
input: frames,
|
|
155
|
+
encoding: "utf8",
|
|
156
|
+
timeout: 20000,
|
|
157
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
158
|
+
});
|
|
159
|
+
if (r.error || !r.stdout) return null;
|
|
160
|
+
for (const line of r.stdout.split("\n")) {
|
|
161
|
+
if (!line.trim()) continue;
|
|
162
|
+
try {
|
|
163
|
+
const msg = JSON.parse(line);
|
|
164
|
+
if (msg.id === 2 && msg.result?.tools) return msg.result.tools.map((t) => t.name);
|
|
165
|
+
} catch {
|
|
166
|
+
/* not a frame */
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return null;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const referencedToolkitTools = refs.get("dev-toolkit");
|
|
173
|
+
let deep = "skipped (no local dev-toolkit checkout)";
|
|
174
|
+
if (referencedToolkitTools) {
|
|
175
|
+
const repo = resolveToolkitRepo();
|
|
176
|
+
const served = repo ? listTools(repo) : null;
|
|
177
|
+
if (!served) {
|
|
178
|
+
deep = repo ? `skipped (${repo} did not answer tools/list)` : deep;
|
|
179
|
+
} else {
|
|
180
|
+
const missing = [...referencedToolkitTools.keys()].filter((t) => !served.includes(t));
|
|
181
|
+
if (missing.length === 0) {
|
|
182
|
+
deep = `all ${referencedToolkitTools.size} referenced tools exist in the live server (${served.length} served)`;
|
|
183
|
+
} else {
|
|
184
|
+
const msg = `dev-toolkit tools referenced but not served (${served.length} served): ${missing
|
|
185
|
+
.map((t) => `${t} [${[...referencedToolkitTools.get(t)][0]}]`)
|
|
186
|
+
.join(", ")}`;
|
|
187
|
+
deep = `${missing.length} referenced tool(s) not served`;
|
|
188
|
+
if (strictTools) errors.push(msg);
|
|
189
|
+
else console.warn(` WARNING - ${msg}`);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const totalRefs = [...refs.values()].reduce((n, tools) => n + tools.size, 0);
|
|
195
|
+
console.log(
|
|
196
|
+
`lint-mcp-refs: ${totalRefs} distinct tool references across ${refs.size} server(s) in ${files.length} files`,
|
|
197
|
+
);
|
|
198
|
+
for (const [server, tools] of refs) {
|
|
199
|
+
console.log(` ${server in KNOWN_SERVERS ? "✓" : "✗"} mcp__${server}__* : ${tools.size} tools`);
|
|
200
|
+
}
|
|
201
|
+
console.log(` tool-existence check: ${deep}`);
|
|
202
|
+
|
|
203
|
+
if (errors.length > 0) {
|
|
204
|
+
console.error(`lint-mcp-refs: ${errors.length} error(s)`);
|
|
205
|
+
for (const e of errors) console.error(" " + e);
|
|
206
|
+
process.exit(1);
|
|
207
|
+
}
|
|
@@ -16,9 +16,13 @@ PROJECT_ROOT="${1:?usage: memory-load.sh <project-root>}"
|
|
|
16
16
|
MEM_DIR="$PROJECT_ROOT/.multi-agent/memory"
|
|
17
17
|
MEM_INDEX="$MEM_DIR/MEMORY.md"
|
|
18
18
|
|
|
19
|
-
# Pref-gate: locate
|
|
19
|
+
# Pref-gate: locate the preferences file (home-dir first, then XDG config).
|
|
20
|
+
# The canonical name written by the installer is multi-agent-preferences.json;
|
|
21
|
+
# the bare preferences.json names are kept only as legacy fallbacks.
|
|
20
22
|
prefs_file=""
|
|
21
23
|
for candidate in \
|
|
24
|
+
"$HOME/.claude/multi-agent-preferences.json" \
|
|
25
|
+
"$HOME/.config/multi-agent-pipeline/multi-agent-preferences.json" \
|
|
22
26
|
"$HOME/.claude/preferences.json" \
|
|
23
27
|
"$HOME/.config/multi-agent-pipeline/preferences.json"
|
|
24
28
|
do
|
|
@@ -126,7 +126,10 @@ fi
|
|
|
126
126
|
changed_block=""
|
|
127
127
|
files_count=0
|
|
128
128
|
total_add=0; total_del=0
|
|
129
|
-
|
|
129
|
+
# Only the directory needs to exist here; `git rev-parse HEAD` below is the real
|
|
130
|
+
# repository check. Testing -d "$WORKTREE/.git" would be wrong regardless: in a
|
|
131
|
+
# linked worktree .git is a file, not a directory.
|
|
132
|
+
if [ -n "$WORKTREE" ] && [ -d "$WORKTREE" ]; then
|
|
130
133
|
if git -C "$WORKTREE" rev-parse HEAD >/dev/null 2>&1; then
|
|
131
134
|
diffstat=$(git -C "$WORKTREE" diff --numstat "$state_base"...HEAD 2>/dev/null || true)
|
|
132
135
|
if [ -n "$diffstat" ]; then
|
|
@@ -82,7 +82,7 @@ function parseArgs(argv) {
|
|
|
82
82
|
// ---- File walking ----------------------------------------------------------
|
|
83
83
|
|
|
84
84
|
const DEFAULT_IGNORE = new Set([
|
|
85
|
-
'.git', 'node_modules', 'Pods', '.build', 'DerivedData', '.next',
|
|
85
|
+
'.git', '.worktrees', 'node_modules', 'Pods', '.build', 'DerivedData', '.next',
|
|
86
86
|
'dist', 'build', '__pycache__', '.venv', 'venv', '.cache',
|
|
87
87
|
'.gradle', '.idea', '.vscode', 'target',
|
|
88
88
|
]);
|