@claude-flow/cli 3.12.0 → 3.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/helpers/auto-memory-hook.mjs +34 -4
- package/.claude/helpers/hook-handler.cjs +17 -13
- package/.claude/helpers/intelligence.cjs +16 -2
- package/README.md +7 -4
- package/dist/src/commands/daemon.d.ts.map +1 -1
- package/dist/src/commands/daemon.js +76 -7
- package/dist/src/commands/daemon.js.map +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/init.js +24 -3
- package/dist/src/commands/init.js.map +1 -1
- package/dist/src/commands/security.d.ts.map +1 -1
- package/dist/src/commands/security.js +70 -12
- package/dist/src/commands/security.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +3 -2
- package/plugins/ruflo-metaharness/.claude-flow/data/pending-insights.jsonl +1 -0
- package/plugins/ruflo-metaharness/.claude-flow/neural/stats.json +6 -0
- package/plugins/ruflo-metaharness/.claude-plugin/plugin.json +32 -0
- package/plugins/ruflo-metaharness/README.md +64 -0
- package/plugins/ruflo-metaharness/agents/metaharness-architect.md +58 -0
- package/plugins/ruflo-metaharness/commands/ruflo-metaharness.md +46 -0
- package/plugins/ruflo-metaharness/scripts/_harness.mjs +261 -0
- package/plugins/ruflo-metaharness/scripts/_similarity.mjs +161 -0
- package/plugins/ruflo-metaharness/scripts/_spike-similarity.mjs +223 -0
- package/plugins/ruflo-metaharness/scripts/audit-list.mjs +158 -0
- package/plugins/ruflo-metaharness/scripts/audit-trend.mjs +272 -0
- package/plugins/ruflo-metaharness/scripts/bench-parse-mcp-scan.mjs +146 -0
- package/plugins/ruflo-metaharness/scripts/bench-recordpair-overhead.mjs +186 -0
- package/plugins/ruflo-metaharness/scripts/bench-similarity.mjs +177 -0
- package/plugins/ruflo-metaharness/scripts/drift-from-history.mjs +363 -0
- package/plugins/ruflo-metaharness/scripts/genome.mjs +80 -0
- package/plugins/ruflo-metaharness/scripts/mcp-scan.mjs +111 -0
- package/plugins/ruflo-metaharness/scripts/mint.mjs +119 -0
- package/plugins/ruflo-metaharness/scripts/oia-audit.mjs +228 -0
- package/plugins/ruflo-metaharness/scripts/router-parallel-analyze.mjs +250 -0
- package/plugins/ruflo-metaharness/scripts/score.mjs +92 -0
- package/plugins/ruflo-metaharness/scripts/similarity.mjs +158 -0
- package/plugins/ruflo-metaharness/scripts/smoke.sh +2268 -0
- package/plugins/ruflo-metaharness/scripts/test-graceful-degradation.mjs +141 -0
- package/plugins/ruflo-metaharness/scripts/test-mcp-tools.mjs +437 -0
- package/plugins/ruflo-metaharness/scripts/test-parallel-pipeline.mjs +204 -0
- package/plugins/ruflo-metaharness/scripts/test-pipeline-roundtrip.mjs +586 -0
- package/plugins/ruflo-metaharness/scripts/test-similarity.mjs +334 -0
- package/plugins/ruflo-metaharness/scripts/test-with-openrouter.mjs +229 -0
- package/plugins/ruflo-metaharness/scripts/threat-model.mjs +59 -0
- package/plugins/ruflo-metaharness/skills/harness-drift-from-history/SKILL.md +65 -0
- package/plugins/ruflo-metaharness/skills/harness-genome/SKILL.md +54 -0
- package/plugins/ruflo-metaharness/skills/harness-mcp-scan/SKILL.md +47 -0
- package/plugins/ruflo-metaharness/skills/harness-mint/SKILL.md +72 -0
- package/plugins/ruflo-metaharness/skills/harness-oia-audit/SKILL.md +79 -0
- package/plugins/ruflo-metaharness/skills/harness-score/SKILL.md +66 -0
- package/plugins/ruflo-metaharness/skills/harness-similarity/SKILL.md +67 -0
- package/plugins/ruflo-metaharness/skills/harness-threat-model/SKILL.md +39 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// oia-audit.mjs — composite Phase-2 worker (ADR-150).
|
|
3
|
+
//
|
|
4
|
+
// Bundles three MetaHarness static-analysis surfaces into one timestamped
|
|
5
|
+
// audit record:
|
|
6
|
+
// - harness oia-manifest (Open Infrastructure Architecture L1-L9 alignment)
|
|
7
|
+
// - harness threat-model (categorized MCP-surface threat report)
|
|
8
|
+
// - harness mcp-scan (per-server/tool policy + permissions + deps)
|
|
9
|
+
//
|
|
10
|
+
// The combined record is stored in the `metaharness-audit` memory
|
|
11
|
+
// namespace, keyed by ISO timestamp. Designed to be invoked on a cron
|
|
12
|
+
// schedule (e.g. weekly) so audit drift is visible over time.
|
|
13
|
+
//
|
|
14
|
+
// USAGE
|
|
15
|
+
// node scripts/oia-audit.mjs # run + store
|
|
16
|
+
// node scripts/oia-audit.mjs --path <dir> # audit specific dir
|
|
17
|
+
// node scripts/oia-audit.mjs --dry-run # don't write to memory
|
|
18
|
+
// node scripts/oia-audit.mjs --alert-on-worst high
|
|
19
|
+
// # exit 1 if threat-model worst >= high
|
|
20
|
+
// node scripts/oia-audit.mjs --format json
|
|
21
|
+
//
|
|
22
|
+
// EXIT CODES
|
|
23
|
+
// 0 audit OK (or degraded)
|
|
24
|
+
// 1 --alert-on-worst threshold exceeded
|
|
25
|
+
// 2 config error or audit failure
|
|
26
|
+
|
|
27
|
+
import { spawnSync } from 'node:child_process';
|
|
28
|
+
// iter 63 — SEVERITY_RANK + rankSeverity consolidated to _harness.mjs
|
|
29
|
+
// (was local in iter 62; now shared with audit-trend + mcp-scan).
|
|
30
|
+
import { runHarness, runMetaharness, runHarnessAsync, runMetaharnessAsync, emitDegradedJsonAndExit, parseMcpScanText, SEVERITY_RANK, rankSeverity } from './_harness.mjs';
|
|
31
|
+
|
|
32
|
+
// iter 63 — SEVERITY_RANK + rankSeverity moved to _harness.mjs (single
|
|
33
|
+
// source of truth). The local copy from iter 62 is gone; the imports
|
|
34
|
+
// at the top of this file provide the same names.
|
|
35
|
+
const NS = process.env.OIA_AUDIT_NAMESPACE || 'metaharness-audit';
|
|
36
|
+
const CLI_PKG = process.env.CLI_CORE === '1'
|
|
37
|
+
? '@claude-flow/cli-core@alpha'
|
|
38
|
+
: '@claude-flow/cli@latest';
|
|
39
|
+
|
|
40
|
+
const ARGS = (() => {
|
|
41
|
+
const a = { path: '.', format: 'json', dryRun: false, alertWorst: null };
|
|
42
|
+
for (let i = 2; i < process.argv.length; i++) {
|
|
43
|
+
const v = process.argv[i];
|
|
44
|
+
if (v === '--path') a.path = process.argv[++i];
|
|
45
|
+
else if (v === '--dry-run') a.dryRun = true;
|
|
46
|
+
else if (v === '--alert-on-worst') a.alertWorst = String(process.argv[++i] || '').toLowerCase();
|
|
47
|
+
else if (v === '--format') a.format = process.argv[++i];
|
|
48
|
+
}
|
|
49
|
+
return a;
|
|
50
|
+
})();
|
|
51
|
+
|
|
52
|
+
// iter 47 — the `harness` and `metaharness` CLIs both have `score` /
|
|
53
|
+
// `genome` subcommands but with DIFFERENT output schemas. For the
|
|
54
|
+
// fingerprint to be consumable by _similarity.mjs (which expects the
|
|
55
|
+
// metaharness CLI shape: harnessFit/compileConfidence/agent_topology),
|
|
56
|
+
// we need to dispatch to runMetaharness for score+genome, not runHarness.
|
|
57
|
+
// iter 56 — annotated runOne result is identical for sync/async paths.
|
|
58
|
+
function annotateOne(r, label) {
|
|
59
|
+
let json = r.json;
|
|
60
|
+
if (label === 'mcp-scan' && !json && !r.degraded && r.stdout) {
|
|
61
|
+
const parsed = parseMcpScanText(r.stdout);
|
|
62
|
+
json = { findings: parsed.findings, summary: parsed.summary, rawStdout: r.stdout.slice(0, 400) };
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
label,
|
|
66
|
+
exitCode: r.exitCode,
|
|
67
|
+
degraded: r.degraded,
|
|
68
|
+
reason: r.degraded ? r.reason : null,
|
|
69
|
+
json,
|
|
70
|
+
durationMs: r.durationMs,
|
|
71
|
+
stderrTail: r.degraded ? (r.stderr || '').slice(-200) : null,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function runOne(args, label, engine = 'harness') {
|
|
76
|
+
const r = engine === 'metaharness' ? runMetaharness(args) : runHarness(args);
|
|
77
|
+
return annotateOne(r, label);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// iter 56 — parallel variant. The composite audit's 5 subprocess calls
|
|
81
|
+
// are mutually independent (each scans the same path read-only), so
|
|
82
|
+
// they can run concurrently. Worst-case wall-clock drops from
|
|
83
|
+
// 5×DEFAULT_TIMEOUT_MS (sequential) to 1×DEFAULT_TIMEOUT_MS (parallel)
|
|
84
|
+
// — the iter-55-flagged gap (oia-audit timeout under unreachable
|
|
85
|
+
// registry) is closed by this.
|
|
86
|
+
async function runAllParallel(path) {
|
|
87
|
+
const tasks = [
|
|
88
|
+
runHarnessAsync(['oia-manifest', path]).then((r) => ['oiaManifest', annotateOne(r, 'oia-manifest')]),
|
|
89
|
+
runHarnessAsync(['threat-model', path]).then((r) => ['threatModel', annotateOne(r, 'threat-model')]),
|
|
90
|
+
runHarnessAsync(['mcp-scan', path]).then((r) => ['mcpScan', annotateOne(r, 'mcp-scan')]),
|
|
91
|
+
runMetaharnessAsync(['score', path]).then((r) => ['score', annotateOne(r, 'score')]),
|
|
92
|
+
runMetaharnessAsync(['genome', path]).then((r) => ['genome', annotateOne(r, 'genome')]),
|
|
93
|
+
];
|
|
94
|
+
const results = await Promise.all(tasks);
|
|
95
|
+
return Object.fromEntries(results);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function persist(payload) {
|
|
99
|
+
const key = `audit-${new Date().toISOString().replace(/[:.]/g, '-')}`;
|
|
100
|
+
const r = spawnSync('npx', [
|
|
101
|
+
CLI_PKG, 'memory', 'store',
|
|
102
|
+
'--namespace', NS,
|
|
103
|
+
'--key', key,
|
|
104
|
+
'--value', JSON.stringify(payload),
|
|
105
|
+
], { stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8', shell: process.platform === 'win32' });
|
|
106
|
+
return {
|
|
107
|
+
ok: r.status === 0,
|
|
108
|
+
namespace: NS,
|
|
109
|
+
key,
|
|
110
|
+
error: r.status === 0 ? null : (r.stderr || '').slice(0, 200),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
async function main() {
|
|
115
|
+
if (ARGS.alertWorst !== null && !SEVERITY_RANK.hasOwnProperty(ARGS.alertWorst)) {
|
|
116
|
+
console.error(`oia-audit: --alert-on-worst must be one of clean|low|medium|high; got ${ARGS.alertWorst}`);
|
|
117
|
+
process.exit(2);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const startedAt = new Date().toISOString();
|
|
121
|
+
const wallStart = Date.now();
|
|
122
|
+
// iter 56 — run the 5 sub-audits in parallel (was sequential pre-iter-56).
|
|
123
|
+
// Worst-case wall-clock improves from 5×TIMEOUT to 1×TIMEOUT in the
|
|
124
|
+
// unreachable-registry case; the happy path improves from sum-of-durations
|
|
125
|
+
// to max-of-durations (typically ~2-4× faster).
|
|
126
|
+
const all = await runAllParallel(ARGS.path);
|
|
127
|
+
const wallMs = Date.now() - wallStart;
|
|
128
|
+
const { oiaManifest: oia, threatModel: tm, mcpScan: mcp, score, genome } = all;
|
|
129
|
+
|
|
130
|
+
// If all FIVE say "metaharness not available", surface the degraded
|
|
131
|
+
// payload exactly once and exit 0 (architectural constraint #3).
|
|
132
|
+
if (oia.degraded && tm.degraded && mcp.degraded && score.degraded && genome.degraded) {
|
|
133
|
+
emitDegradedJsonAndExit('metaharness-not-available');
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Aggregate the worst-severity signal across mcp-scan + threat-model.
|
|
138
|
+
// iter 62 — safe lookup using `?? 0` so unknown severities don't
|
|
139
|
+
// silently bump composite via NaN-comparison (was: `acc | undefined`
|
|
140
|
+
// → reduce never updated). Now any [WARN]/[CRITICAL]/[ERROR] finding
|
|
141
|
+
// correctly elevates composite worst.
|
|
142
|
+
const tmWorst = String(tm.json?.worst || 'clean').toLowerCase();
|
|
143
|
+
const mcpFindings = Array.isArray(mcp.json?.findings) ? mcp.json.findings : [];
|
|
144
|
+
// iter 63 — rankSeverity() from _harness.mjs handles unknown-severity
|
|
145
|
+
// safely (returns 0 instead of undefined) so the reduce never sees NaN.
|
|
146
|
+
const mcpWorst = mcpFindings.reduce((acc, f) => {
|
|
147
|
+
const s = String(f.severity || 'low').toLowerCase();
|
|
148
|
+
return rankSeverity(s) > rankSeverity(acc) ? s : acc;
|
|
149
|
+
}, 'clean');
|
|
150
|
+
const compositeWorst = rankSeverity(tmWorst) > rankSeverity(mcpWorst) ? tmWorst : mcpWorst;
|
|
151
|
+
|
|
152
|
+
let alertTriggered = false;
|
|
153
|
+
let alertReason = null;
|
|
154
|
+
if (ARGS.alertWorst !== null) {
|
|
155
|
+
const threshold = rankSeverity(ARGS.alertWorst);
|
|
156
|
+
const compositeRank = rankSeverity(compositeWorst);
|
|
157
|
+
if (compositeRank >= threshold && threshold > 0) {
|
|
158
|
+
alertTriggered = true;
|
|
159
|
+
alertReason = `composite worst=${compositeWorst} ≥ ${ARGS.alertWorst}`;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// iter 59 — surface parallelization metrics. wallMs is the actual
|
|
164
|
+
// time the 5 subprocess race took (max of components); the sum of
|
|
165
|
+
// component.durationMs is what a SERIAL implementation would have
|
|
166
|
+
// taken. A future smoke gate compares them to catch silent
|
|
167
|
+
// serialization regression.
|
|
168
|
+
const sumComponentMs = Object.values(all).reduce(
|
|
169
|
+
(a, c) => a + (c?.durationMs ?? 0), 0);
|
|
170
|
+
const payload = {
|
|
171
|
+
path: ARGS.path,
|
|
172
|
+
startedAt,
|
|
173
|
+
finishedAt: new Date().toISOString(),
|
|
174
|
+
timing: {
|
|
175
|
+
wallMs,
|
|
176
|
+
sumComponentMs,
|
|
177
|
+
parallelSpeedup: sumComponentMs > 0
|
|
178
|
+
? Math.round((sumComponentMs / Math.max(wallMs, 1)) * 100) / 100 : 0,
|
|
179
|
+
},
|
|
180
|
+
composite: { worst: compositeWorst, threatModelWorst: tmWorst, mcpScanWorst: mcpWorst },
|
|
181
|
+
components: { oiaManifest: oia, threatModel: tm, mcpScan: mcp, score, genome },
|
|
182
|
+
// iter 38 — denormalized harness fingerprint for cheap similarity().
|
|
183
|
+
// Mirrors the shape `_similarity.mjs::similarity()` expects so
|
|
184
|
+
// audit-trend can call it without reshuffling components.
|
|
185
|
+
fingerprint: {
|
|
186
|
+
score: score?.json && !score.degraded ? score.json : null,
|
|
187
|
+
genome: genome?.json && !genome.degraded ? genome.json : null,
|
|
188
|
+
},
|
|
189
|
+
alert: ARGS.alertWorst !== null ? {
|
|
190
|
+
threshold: ARGS.alertWorst,
|
|
191
|
+
triggered: alertTriggered,
|
|
192
|
+
reason: alertReason || `composite worst=${compositeWorst} < ${ARGS.alertWorst} — OK`,
|
|
193
|
+
} : null,
|
|
194
|
+
persisted: null,
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
if (!ARGS.dryRun) {
|
|
198
|
+
payload.persisted = persist(payload);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
if (ARGS.format === 'json') {
|
|
202
|
+
console.log(JSON.stringify(payload, null, 2));
|
|
203
|
+
} else {
|
|
204
|
+
console.log(`# oia-audit — ${ARGS.path}`);
|
|
205
|
+
console.log('');
|
|
206
|
+
console.log(`| Component | Exit | Degraded | Duration |`);
|
|
207
|
+
console.log(`|---|---:|:---:|---:|`);
|
|
208
|
+
console.log(`| oia-manifest | ${oia.exitCode} | ${oia.degraded ? '⚠' : '✓'} | ${oia.durationMs}ms |`);
|
|
209
|
+
console.log(`| threat-model | ${tm.exitCode} | ${tm.degraded ? '⚠' : '✓'} | ${tm.durationMs}ms |`);
|
|
210
|
+
console.log(`| mcp-scan | ${mcp.exitCode} | ${mcp.degraded ? '⚠' : '✓'} | ${mcp.durationMs}ms |`);
|
|
211
|
+
console.log('');
|
|
212
|
+
console.log(`Composite worst severity: **${compositeWorst}** (tm=${tmWorst}, mcp=${mcpWorst})`);
|
|
213
|
+
if (payload.persisted) {
|
|
214
|
+
console.log(`Persisted: ${payload.persisted.ok ? `${payload.persisted.namespace}:${payload.persisted.key}` : `FAILED: ${payload.persisted.error}`}`);
|
|
215
|
+
}
|
|
216
|
+
if (payload.alert) {
|
|
217
|
+
console.log('');
|
|
218
|
+
console.log(payload.alert.triggered ? `⚠ **ALERT**: ${payload.alert.reason}` : `✓ ${payload.alert.reason}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
if (alertTriggered) process.exit(1);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
main().catch((e) => {
|
|
226
|
+
console.error('oia-audit crashed:', e?.message ?? e);
|
|
227
|
+
process.exit(2);
|
|
228
|
+
});
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// router-parallel-analyze.mjs — ADR-150 Phase 2 SelfEvolvingRouter
|
|
3
|
+
// promotion gate.
|
|
4
|
+
//
|
|
5
|
+
// Reads paired routing decisions (Thompson bandit pick + SelfEvolvingRouter
|
|
6
|
+
// pick, both with measured outcomes) from a JSONL trajectory file and
|
|
7
|
+
// computes the THREE PROMOTION CRITERIA from ADR-150 review-round-1:
|
|
8
|
+
//
|
|
9
|
+
// (a) qualityScore improvement > 2% (relative)
|
|
10
|
+
// (b) usdPerDecision increase < 1% (relative)
|
|
11
|
+
// (c) p95 routing-decision latency increase < 5% (relative)
|
|
12
|
+
//
|
|
13
|
+
// ALL THREE must hold for promotion. AND, not OR — the OR form let
|
|
14
|
+
// quality gains mask cost regressions, exactly the failure mode ADR-149's
|
|
15
|
+
// Pareto framing was built to prevent.
|
|
16
|
+
//
|
|
17
|
+
// INPUT FORMAT (.swarm/router-parallel.jsonl, one JSON record per line):
|
|
18
|
+
// {
|
|
19
|
+
// "ts": "<iso>",
|
|
20
|
+
// "task": { ... },
|
|
21
|
+
// "bandit": { "pick": "<modelId>", "predictedQuality": 0.84, "predictedCostUsd": 0.003 },
|
|
22
|
+
// "ser": { "pick": "<modelId>", "predictedQuality": 0.87, "predictedCostUsd": 0.004 },
|
|
23
|
+
// "outcome": { "actualModel": "<modelId>", "actualQuality": 0.91, "actualUsd": 0.0035, "actualLatencyMs": 1240 }
|
|
24
|
+
// }
|
|
25
|
+
//
|
|
26
|
+
// USAGE
|
|
27
|
+
// node scripts/router-parallel-analyze.mjs --input .swarm/router-parallel.jsonl
|
|
28
|
+
// node scripts/router-parallel-analyze.mjs --input <file> --format json
|
|
29
|
+
// node scripts/router-parallel-analyze.mjs --input <file> --strict # exit 1 if NOT promotable
|
|
30
|
+
//
|
|
31
|
+
// EXIT CODES
|
|
32
|
+
// 0 analysis complete; in --strict mode → all three criteria passed
|
|
33
|
+
// 1 --strict requested AND at least one criterion failed (NOT promotable)
|
|
34
|
+
// 2 config error or input file missing/malformed
|
|
35
|
+
|
|
36
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
37
|
+
|
|
38
|
+
const ARGS = (() => {
|
|
39
|
+
const a = { input: '.swarm/router-parallel.jsonl', format: 'table', strict: false };
|
|
40
|
+
for (let i = 2; i < process.argv.length; i++) {
|
|
41
|
+
const v = process.argv[i];
|
|
42
|
+
if (v === '--input') a.input = process.argv[++i];
|
|
43
|
+
else if (v === '--format') a.format = process.argv[++i];
|
|
44
|
+
else if (v === '--strict') a.strict = true;
|
|
45
|
+
}
|
|
46
|
+
return a;
|
|
47
|
+
})();
|
|
48
|
+
|
|
49
|
+
function median(sorted) {
|
|
50
|
+
if (sorted.length === 0) return 0;
|
|
51
|
+
const n = sorted.length;
|
|
52
|
+
return n % 2 === 0 ? (sorted[n / 2 - 1] + sorted[n / 2]) / 2 : sorted[(n - 1) / 2];
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function pctile(sorted, q) {
|
|
56
|
+
if (sorted.length === 0) return 0;
|
|
57
|
+
const idx = Math.floor(q * (sorted.length - 1));
|
|
58
|
+
return sorted[idx];
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function mean(arr) {
|
|
62
|
+
if (arr.length === 0) return 0;
|
|
63
|
+
return arr.reduce((s, x) => s + x, 0) / arr.length;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function main() {
|
|
67
|
+
if (!existsSync(ARGS.input)) {
|
|
68
|
+
if (ARGS.format === 'json') {
|
|
69
|
+
console.log(JSON.stringify({
|
|
70
|
+
ok: true,
|
|
71
|
+
sufficient: false,
|
|
72
|
+
reason: 'input-file-not-found',
|
|
73
|
+
input: ARGS.input,
|
|
74
|
+
hint: 'Enable parallel logging (CLAUDE_FLOW_ROUTER_PARALLEL_LOG=1) and run a workload first.',
|
|
75
|
+
}, null, 2));
|
|
76
|
+
} else {
|
|
77
|
+
console.log(`# router-parallel-analyze`);
|
|
78
|
+
console.log('');
|
|
79
|
+
console.log(`_Input file not found: ${ARGS.input}_`);
|
|
80
|
+
console.log('');
|
|
81
|
+
console.log('Enable parallel logging by setting `CLAUDE_FLOW_ROUTER_PARALLEL_LOG=1`');
|
|
82
|
+
console.log('and running a real workload, then re-run this analyzer.');
|
|
83
|
+
}
|
|
84
|
+
process.exit(ARGS.strict ? 1 : 0);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
let rows = [];
|
|
88
|
+
try {
|
|
89
|
+
rows = readFileSync(ARGS.input, 'utf-8')
|
|
90
|
+
.split('\n')
|
|
91
|
+
.filter((l) => l.trim())
|
|
92
|
+
.map((l) => JSON.parse(l));
|
|
93
|
+
} catch (e) {
|
|
94
|
+
console.error(`router-parallel-analyze: cannot parse ${ARGS.input}: ${e.message}`);
|
|
95
|
+
process.exit(2);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Filter to records with both predictions + outcome present.
|
|
99
|
+
const usable = rows.filter((r) => r.bandit && r.ser && r.outcome);
|
|
100
|
+
|
|
101
|
+
if (usable.length < 30) {
|
|
102
|
+
const payload = {
|
|
103
|
+
ok: true,
|
|
104
|
+
sufficient: false,
|
|
105
|
+
reason: `n=${usable.length} < 30 (insufficient sample for AND-gate evaluation)`,
|
|
106
|
+
hint: 'Continue collecting parallel-routing data; statistical significance needs ≥30 paired decisions per arm.',
|
|
107
|
+
sampleSize: usable.length,
|
|
108
|
+
generatedAt: new Date().toISOString(),
|
|
109
|
+
};
|
|
110
|
+
if (ARGS.format === 'json') console.log(JSON.stringify(payload, null, 2));
|
|
111
|
+
else {
|
|
112
|
+
console.log(`# router-parallel-analyze`);
|
|
113
|
+
console.log('');
|
|
114
|
+
console.log(`Sample size: **${usable.length}** paired decisions`);
|
|
115
|
+
console.log('');
|
|
116
|
+
console.log(`_Insufficient — need ≥30 for the 3-criteria AND-gate._`);
|
|
117
|
+
}
|
|
118
|
+
process.exit(ARGS.strict ? 1 : 0);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// For each record, attribute the outcome to BOTH arms when they
|
|
122
|
+
// agreed on the model pick; when they disagreed, the actual outcome
|
|
123
|
+
// belongs only to the bandit (which was the executing arm). The SER
|
|
124
|
+
// arm gets its PREDICTED metrics for the counterfactual.
|
|
125
|
+
const banditOutcomes = usable.map((r) => ({
|
|
126
|
+
qualityActual: r.outcome.actualQuality,
|
|
127
|
+
usdActual: r.outcome.actualUsd,
|
|
128
|
+
latencyActual: r.outcome.actualLatencyMs,
|
|
129
|
+
}));
|
|
130
|
+
|
|
131
|
+
// SER counterfactual: use ser.predictedQuality / predictedCostUsd
|
|
132
|
+
// when picks disagreed; use actual when they agreed. Latency: when
|
|
133
|
+
// they agreed, both saw the same actual latency; when they
|
|
134
|
+
// disagreed, the SER pick's latency is unknown — fall back to the
|
|
135
|
+
// mean of agreed-decision latencies for that model.
|
|
136
|
+
const agreedLatencyByModel = {};
|
|
137
|
+
for (const r of usable) {
|
|
138
|
+
if (r.bandit.pick === r.ser.pick) {
|
|
139
|
+
const m = r.outcome.actualModel || r.bandit.pick;
|
|
140
|
+
(agreedLatencyByModel[m] = agreedLatencyByModel[m] || []).push(r.outcome.actualLatencyMs);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
const modelMeanLatency = {};
|
|
144
|
+
for (const [m, arr] of Object.entries(agreedLatencyByModel)) {
|
|
145
|
+
modelMeanLatency[m] = mean(arr);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const serOutcomes = usable.map((r) => {
|
|
149
|
+
const agreed = r.bandit.pick === r.ser.pick;
|
|
150
|
+
return {
|
|
151
|
+
qualityActual: agreed ? r.outcome.actualQuality : r.ser.predictedQuality,
|
|
152
|
+
usdActual: agreed ? r.outcome.actualUsd : r.ser.predictedCostUsd,
|
|
153
|
+
latencyActual: agreed
|
|
154
|
+
? r.outcome.actualLatencyMs
|
|
155
|
+
: (modelMeanLatency[r.ser.pick] ?? r.outcome.actualLatencyMs),
|
|
156
|
+
};
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
// Compute the three criteria.
|
|
160
|
+
const banditQuality = mean(banditOutcomes.map((o) => o.qualityActual));
|
|
161
|
+
const serQuality = mean(serOutcomes.map((o) => o.qualityActual));
|
|
162
|
+
const qualityImprovementPct =
|
|
163
|
+
banditQuality > 0 ? ((serQuality - banditQuality) / banditQuality) * 100 : 0;
|
|
164
|
+
|
|
165
|
+
const banditUsd = mean(banditOutcomes.map((o) => o.usdActual));
|
|
166
|
+
const serUsd = mean(serOutcomes.map((o) => o.usdActual));
|
|
167
|
+
const usdIncreasePct = banditUsd > 0 ? ((serUsd - banditUsd) / banditUsd) * 100 : 0;
|
|
168
|
+
|
|
169
|
+
const banditLatencyP95 = pctile(banditOutcomes.map((o) => o.latencyActual).sort((a, b) => a - b), 0.95);
|
|
170
|
+
const serLatencyP95 = pctile(serOutcomes.map((o) => o.latencyActual).sort((a, b) => a - b), 0.95);
|
|
171
|
+
const latencyIncreasePct =
|
|
172
|
+
banditLatencyP95 > 0 ? ((serLatencyP95 - banditLatencyP95) / banditLatencyP95) * 100 : 0;
|
|
173
|
+
|
|
174
|
+
const passes = {
|
|
175
|
+
quality: qualityImprovementPct > 2,
|
|
176
|
+
cost: usdIncreasePct < 1,
|
|
177
|
+
latency: latencyIncreasePct < 5,
|
|
178
|
+
};
|
|
179
|
+
const allPass = passes.quality && passes.cost && passes.latency;
|
|
180
|
+
|
|
181
|
+
// Disagreement rate (informational; not part of the gate).
|
|
182
|
+
const disagreements = usable.filter((r) => r.bandit.pick !== r.ser.pick).length;
|
|
183
|
+
const disagreementPct = (disagreements / usable.length) * 100;
|
|
184
|
+
|
|
185
|
+
const payload = {
|
|
186
|
+
ok: true,
|
|
187
|
+
sufficient: true,
|
|
188
|
+
sampleSize: usable.length,
|
|
189
|
+
disagreement: {
|
|
190
|
+
count: disagreements,
|
|
191
|
+
pct: Math.round(disagreementPct * 100) / 100,
|
|
192
|
+
},
|
|
193
|
+
bandit: {
|
|
194
|
+
meanQuality: Math.round(banditQuality * 10000) / 10000,
|
|
195
|
+
meanUsd: Math.round(banditUsd * 1e6) / 1e6,
|
|
196
|
+
p95LatencyMs: Math.round(banditLatencyP95),
|
|
197
|
+
},
|
|
198
|
+
ser: {
|
|
199
|
+
meanQuality: Math.round(serQuality * 10000) / 10000,
|
|
200
|
+
meanUsd: Math.round(serUsd * 1e6) / 1e6,
|
|
201
|
+
p95LatencyMs: Math.round(serLatencyP95),
|
|
202
|
+
},
|
|
203
|
+
criteria: {
|
|
204
|
+
qualityImprovementPct: Math.round(qualityImprovementPct * 100) / 100,
|
|
205
|
+
qualityThresholdPct: 2,
|
|
206
|
+
qualityPasses: passes.quality,
|
|
207
|
+
usdIncreasePct: Math.round(usdIncreasePct * 100) / 100,
|
|
208
|
+
usdThresholdPct: 1,
|
|
209
|
+
costPasses: passes.cost,
|
|
210
|
+
latencyIncreasePct: Math.round(latencyIncreasePct * 100) / 100,
|
|
211
|
+
latencyThresholdPct: 5,
|
|
212
|
+
latencyPasses: passes.latency,
|
|
213
|
+
},
|
|
214
|
+
verdict: {
|
|
215
|
+
promotable: allPass,
|
|
216
|
+
reason: allPass
|
|
217
|
+
? 'All three criteria met (quality > 2% AND cost < 1% AND latency < 5%)'
|
|
218
|
+
: `Blocked by: ${[
|
|
219
|
+
!passes.quality && `quality only +${(qualityImprovementPct).toFixed(2)}% (need >2%)`,
|
|
220
|
+
!passes.cost && `cost +${(usdIncreasePct).toFixed(2)}% (need <1%)`,
|
|
221
|
+
!passes.latency && `latency +${(latencyIncreasePct).toFixed(2)}% (need <5%)`,
|
|
222
|
+
].filter(Boolean).join(', ')}`,
|
|
223
|
+
},
|
|
224
|
+
generatedAt: new Date().toISOString(),
|
|
225
|
+
};
|
|
226
|
+
|
|
227
|
+
if (ARGS.format === 'json') {
|
|
228
|
+
console.log(JSON.stringify(payload, null, 2));
|
|
229
|
+
} else {
|
|
230
|
+
console.log('# router-parallel-analyze (ADR-150 SelfEvolvingRouter promotion gate)');
|
|
231
|
+
console.log('');
|
|
232
|
+
console.log(`Sample size: **${payload.sampleSize}** paired decisions`);
|
|
233
|
+
console.log(`Disagreement rate: ${payload.disagreement.pct}% (${payload.disagreement.count} / ${payload.sampleSize})`);
|
|
234
|
+
console.log('');
|
|
235
|
+
console.log(`| Metric | Bandit | SER | Delta | Threshold | Passes |`);
|
|
236
|
+
console.log(`|---|---:|---:|---:|---:|:---:|`);
|
|
237
|
+
console.log(`| Mean quality | ${payload.bandit.meanQuality} | ${payload.ser.meanQuality} | ${payload.criteria.qualityImprovementPct >= 0 ? '+' : ''}${payload.criteria.qualityImprovementPct}% | >+2% | ${passes.quality ? '✓' : '⚠'} |`);
|
|
238
|
+
console.log(`| Mean $/decision | $${payload.bandit.meanUsd.toFixed(6)} | $${payload.ser.meanUsd.toFixed(6)} | ${payload.criteria.usdIncreasePct >= 0 ? '+' : ''}${payload.criteria.usdIncreasePct}% | <+1% | ${passes.cost ? '✓' : '⚠'} |`);
|
|
239
|
+
console.log(`| p95 latency | ${payload.bandit.p95LatencyMs}ms | ${payload.ser.p95LatencyMs}ms | ${payload.criteria.latencyIncreasePct >= 0 ? '+' : ''}${payload.criteria.latencyIncreasePct}% | <+5% | ${passes.latency ? '✓' : '⚠'} |`);
|
|
240
|
+
console.log('');
|
|
241
|
+
console.log(`**Verdict**: ${allPass ? '✓ PROMOTABLE' : '⚠ NOT promotable'}`);
|
|
242
|
+
console.log('');
|
|
243
|
+
console.log(`_${payload.verdict.reason}_`);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
if (ARGS.strict && !allPass) process.exit(1);
|
|
247
|
+
process.exit(0);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
main();
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// score.mjs — `cost-style` wrapper around `metaharness score <path>`.
|
|
3
|
+
//
|
|
4
|
+
// Returns the 5-dimension scorecard (harnessFit / compileConfidence /
|
|
5
|
+
// taskCoverage / toolSafety / memoryUsefulness) + estCostPerRunUsd +
|
|
6
|
+
// scaffoldReady boolean. Reads-only; no side effects.
|
|
7
|
+
//
|
|
8
|
+
// USAGE
|
|
9
|
+
// node scripts/score.mjs # current dir
|
|
10
|
+
// node scripts/score.mjs --path <dir> # specific dir
|
|
11
|
+
// node scripts/score.mjs --alert-on-fit-below 70 # exit 1 if harnessFit < 70
|
|
12
|
+
// node scripts/score.mjs --format json
|
|
13
|
+
//
|
|
14
|
+
// EXIT CODES
|
|
15
|
+
// 0 scored OK (or degraded — MetaHarness not available)
|
|
16
|
+
// 1 --alert-on-fit-below threshold breached
|
|
17
|
+
// 2 config error or scoring failure
|
|
18
|
+
|
|
19
|
+
import { runMetaharness, emitDegradedJsonAndExit } from './_harness.mjs';
|
|
20
|
+
|
|
21
|
+
const ARGS = (() => {
|
|
22
|
+
const a = { path: '.', format: 'json', alertFitBelow: null };
|
|
23
|
+
for (let i = 2; i < process.argv.length; i++) {
|
|
24
|
+
const v = process.argv[i];
|
|
25
|
+
if (v === '--path') a.path = process.argv[++i];
|
|
26
|
+
else if (v === '--alert-on-fit-below') a.alertFitBelow = parseFloat(process.argv[++i]);
|
|
27
|
+
else if (v === '--format') a.format = process.argv[++i];
|
|
28
|
+
}
|
|
29
|
+
return a;
|
|
30
|
+
})();
|
|
31
|
+
|
|
32
|
+
function main() {
|
|
33
|
+
const r = runMetaharness(['score', ARGS.path]);
|
|
34
|
+
if (r.degraded) {
|
|
35
|
+
emitDegradedJsonAndExit(r.reason);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
if (r.exitCode !== 0 || !r.json) {
|
|
39
|
+
console.error(`score: metaharness exited ${r.exitCode}`);
|
|
40
|
+
if (r.stderr) console.error(r.stderr.slice(0, 400));
|
|
41
|
+
process.exit(2);
|
|
42
|
+
}
|
|
43
|
+
const payload = { ...r.json, durationMs: r.durationMs };
|
|
44
|
+
|
|
45
|
+
if (ARGS.alertFitBelow !== null) {
|
|
46
|
+
if (!isFinite(ARGS.alertFitBelow)) {
|
|
47
|
+
console.error(`score: --alert-on-fit-below must be a finite number`);
|
|
48
|
+
process.exit(2);
|
|
49
|
+
}
|
|
50
|
+
payload.alert = {
|
|
51
|
+
threshold: ARGS.alertFitBelow,
|
|
52
|
+
triggered: typeof payload.harnessFit === 'number' && payload.harnessFit < ARGS.alertFitBelow,
|
|
53
|
+
reason: typeof payload.harnessFit === 'number' && payload.harnessFit < ARGS.alertFitBelow
|
|
54
|
+
? `harnessFit ${payload.harnessFit} < ${ARGS.alertFitBelow}`
|
|
55
|
+
: `harnessFit ${payload.harnessFit ?? 'unknown'} ≥ ${ARGS.alertFitBelow} — OK`,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
if (ARGS.format === 'json') {
|
|
60
|
+
console.log(JSON.stringify(payload, null, 2));
|
|
61
|
+
} else {
|
|
62
|
+
console.log(`# harness-score — ${payload.repo || ARGS.path}`);
|
|
63
|
+
console.log('');
|
|
64
|
+
console.log(`| Dimension | Value |`);
|
|
65
|
+
console.log(`|---|---:|`);
|
|
66
|
+
console.log(`| harnessFit | ${payload.harnessFit ?? '—'} |`);
|
|
67
|
+
console.log(`| compileConfidence | ${payload.compileConfidence ?? '—'} |`);
|
|
68
|
+
console.log(`| taskCoverage | ${payload.taskCoverage ?? '—'} |`);
|
|
69
|
+
console.log(`| toolSafety | ${payload.toolSafety ?? '—'} |`);
|
|
70
|
+
console.log(`| memoryUsefulness | ${payload.memoryUsefulness ?? '—'} |`);
|
|
71
|
+
console.log(`| estCostPerRunUsd | $${payload.estCostPerRunUsd ?? '—'} |`);
|
|
72
|
+
console.log(`| recommendedMode | ${payload.recommendedMode ?? '—'} |`);
|
|
73
|
+
console.log(`| archetype | ${payload.archetype ?? '—'} |`);
|
|
74
|
+
console.log(`| template | ${payload.template ?? '—'} |`);
|
|
75
|
+
console.log(`| scaffoldReady | ${payload.scaffoldReady ?? '—'} |`);
|
|
76
|
+
console.log(`| hardConstraints | ${payload.hardConstraints ?? '—'} |`);
|
|
77
|
+
console.log(`| **duration** | ${payload.durationMs}ms |`);
|
|
78
|
+
console.log('');
|
|
79
|
+
if (payload.alert) {
|
|
80
|
+
if (payload.alert.triggered) {
|
|
81
|
+
console.log(`⚠ **ALERT**: ${payload.alert.reason}`);
|
|
82
|
+
} else {
|
|
83
|
+
console.log(`✓ ${payload.alert.reason}`);
|
|
84
|
+
}
|
|
85
|
+
console.log('');
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (payload.alert?.triggered) process.exit(1);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
main();
|