@claude-flow/cli 3.12.0 → 3.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/package.json +3 -2
  2. package/plugins/ruflo-metaharness/.claude-flow/data/pending-insights.jsonl +1 -0
  3. package/plugins/ruflo-metaharness/.claude-flow/neural/stats.json +6 -0
  4. package/plugins/ruflo-metaharness/.claude-plugin/plugin.json +32 -0
  5. package/plugins/ruflo-metaharness/README.md +64 -0
  6. package/plugins/ruflo-metaharness/agents/metaharness-architect.md +58 -0
  7. package/plugins/ruflo-metaharness/commands/ruflo-metaharness.md +46 -0
  8. package/plugins/ruflo-metaharness/scripts/_harness.mjs +261 -0
  9. package/plugins/ruflo-metaharness/scripts/_similarity.mjs +161 -0
  10. package/plugins/ruflo-metaharness/scripts/_spike-similarity.mjs +223 -0
  11. package/plugins/ruflo-metaharness/scripts/audit-list.mjs +158 -0
  12. package/plugins/ruflo-metaharness/scripts/audit-trend.mjs +272 -0
  13. package/plugins/ruflo-metaharness/scripts/bench-parse-mcp-scan.mjs +146 -0
  14. package/plugins/ruflo-metaharness/scripts/bench-recordpair-overhead.mjs +186 -0
  15. package/plugins/ruflo-metaharness/scripts/bench-similarity.mjs +177 -0
  16. package/plugins/ruflo-metaharness/scripts/drift-from-history.mjs +363 -0
  17. package/plugins/ruflo-metaharness/scripts/genome.mjs +80 -0
  18. package/plugins/ruflo-metaharness/scripts/mcp-scan.mjs +111 -0
  19. package/plugins/ruflo-metaharness/scripts/mint.mjs +119 -0
  20. package/plugins/ruflo-metaharness/scripts/oia-audit.mjs +228 -0
  21. package/plugins/ruflo-metaharness/scripts/router-parallel-analyze.mjs +250 -0
  22. package/plugins/ruflo-metaharness/scripts/score.mjs +92 -0
  23. package/plugins/ruflo-metaharness/scripts/similarity.mjs +158 -0
  24. package/plugins/ruflo-metaharness/scripts/smoke.sh +2268 -0
  25. package/plugins/ruflo-metaharness/scripts/test-graceful-degradation.mjs +141 -0
  26. package/plugins/ruflo-metaharness/scripts/test-mcp-tools.mjs +437 -0
  27. package/plugins/ruflo-metaharness/scripts/test-parallel-pipeline.mjs +204 -0
  28. package/plugins/ruflo-metaharness/scripts/test-pipeline-roundtrip.mjs +586 -0
  29. package/plugins/ruflo-metaharness/scripts/test-similarity.mjs +334 -0
  30. package/plugins/ruflo-metaharness/scripts/test-with-openrouter.mjs +229 -0
  31. package/plugins/ruflo-metaharness/scripts/threat-model.mjs +59 -0
  32. package/plugins/ruflo-metaharness/skills/harness-drift-from-history/SKILL.md +65 -0
  33. package/plugins/ruflo-metaharness/skills/harness-genome/SKILL.md +54 -0
  34. package/plugins/ruflo-metaharness/skills/harness-mcp-scan/SKILL.md +47 -0
  35. package/plugins/ruflo-metaharness/skills/harness-mint/SKILL.md +72 -0
  36. package/plugins/ruflo-metaharness/skills/harness-oia-audit/SKILL.md +79 -0
  37. package/plugins/ruflo-metaharness/skills/harness-score/SKILL.md +66 -0
  38. package/plugins/ruflo-metaharness/skills/harness-similarity/SKILL.md +67 -0
  39. package/plugins/ruflo-metaharness/skills/harness-threat-model/SKILL.md +39 -0
@@ -0,0 +1,228 @@
1
+ #!/usr/bin/env node
2
+ // oia-audit.mjs — composite Phase-2 worker (ADR-150).
3
+ //
4
+ // Bundles three MetaHarness static-analysis surfaces into one timestamped
5
+ // audit record:
6
+ // - harness oia-manifest (Open Infrastructure Architecture L1-L9 alignment)
7
+ // - harness threat-model (categorized MCP-surface threat report)
8
+ // - harness mcp-scan (per-server/tool policy + permissions + deps)
9
+ //
10
+ // The combined record is stored in the `metaharness-audit` memory
11
+ // namespace, keyed by ISO timestamp. Designed to be invoked on a cron
12
+ // schedule (e.g. weekly) so audit drift is visible over time.
13
+ //
14
+ // USAGE
15
+ // node scripts/oia-audit.mjs # run + store
16
+ // node scripts/oia-audit.mjs --path <dir> # audit specific dir
17
+ // node scripts/oia-audit.mjs --dry-run # don't write to memory
18
+ // node scripts/oia-audit.mjs --alert-on-worst high
19
+ // # exit 1 if threat-model worst >= high
20
+ // node scripts/oia-audit.mjs --format json
21
+ //
22
+ // EXIT CODES
23
+ // 0 audit OK (or degraded)
24
+ // 1 --alert-on-worst threshold exceeded
25
+ // 2 config error or audit failure
26
+
27
+ import { spawnSync } from 'node:child_process';
28
+ // iter 63 — SEVERITY_RANK + rankSeverity consolidated to _harness.mjs
29
+ // (was local in iter 62; now shared with audit-trend + mcp-scan).
30
+ import { runHarness, runMetaharness, runHarnessAsync, runMetaharnessAsync, emitDegradedJsonAndExit, parseMcpScanText, SEVERITY_RANK, rankSeverity } from './_harness.mjs';
31
+
32
+ // iter 63 — SEVERITY_RANK + rankSeverity moved to _harness.mjs (single
33
+ // source of truth). The local copy from iter 62 is gone; the imports
34
+ // at the top of this file provide the same names.
35
+ const NS = process.env.OIA_AUDIT_NAMESPACE || 'metaharness-audit';
36
+ const CLI_PKG = process.env.CLI_CORE === '1'
37
+ ? '@claude-flow/cli-core@alpha'
38
+ : '@claude-flow/cli@latest';
39
+
40
+ const ARGS = (() => {
41
+ const a = { path: '.', format: 'json', dryRun: false, alertWorst: null };
42
+ for (let i = 2; i < process.argv.length; i++) {
43
+ const v = process.argv[i];
44
+ if (v === '--path') a.path = process.argv[++i];
45
+ else if (v === '--dry-run') a.dryRun = true;
46
+ else if (v === '--alert-on-worst') a.alertWorst = String(process.argv[++i] || '').toLowerCase();
47
+ else if (v === '--format') a.format = process.argv[++i];
48
+ }
49
+ return a;
50
+ })();
51
+
52
+ // iter 47 — the `harness` and `metaharness` CLIs both have `score` /
53
+ // `genome` subcommands but with DIFFERENT output schemas. For the
54
+ // fingerprint to be consumable by _similarity.mjs (which expects the
55
+ // metaharness CLI shape: harnessFit/compileConfidence/agent_topology),
56
+ // we need to dispatch to runMetaharness for score+genome, not runHarness.
57
+ // iter 56 — annotated runOne result is identical for sync/async paths.
58
+ function annotateOne(r, label) {
59
+ let json = r.json;
60
+ if (label === 'mcp-scan' && !json && !r.degraded && r.stdout) {
61
+ const parsed = parseMcpScanText(r.stdout);
62
+ json = { findings: parsed.findings, summary: parsed.summary, rawStdout: r.stdout.slice(0, 400) };
63
+ }
64
+ return {
65
+ label,
66
+ exitCode: r.exitCode,
67
+ degraded: r.degraded,
68
+ reason: r.degraded ? r.reason : null,
69
+ json,
70
+ durationMs: r.durationMs,
71
+ stderrTail: r.degraded ? (r.stderr || '').slice(-200) : null,
72
+ };
73
+ }
74
+
75
+ function runOne(args, label, engine = 'harness') {
76
+ const r = engine === 'metaharness' ? runMetaharness(args) : runHarness(args);
77
+ return annotateOne(r, label);
78
+ }
79
+
80
+ // iter 56 — parallel variant. The composite audit's 5 subprocess calls
81
+ // are mutually independent (each scans the same path read-only), so
82
+ // they can run concurrently. Worst-case wall-clock drops from
83
+ // 5×DEFAULT_TIMEOUT_MS (sequential) to 1×DEFAULT_TIMEOUT_MS (parallel)
84
+ // — the iter-55-flagged gap (oia-audit timeout under unreachable
85
+ // registry) is closed by this.
86
+ async function runAllParallel(path) {
87
+ const tasks = [
88
+ runHarnessAsync(['oia-manifest', path]).then((r) => ['oiaManifest', annotateOne(r, 'oia-manifest')]),
89
+ runHarnessAsync(['threat-model', path]).then((r) => ['threatModel', annotateOne(r, 'threat-model')]),
90
+ runHarnessAsync(['mcp-scan', path]).then((r) => ['mcpScan', annotateOne(r, 'mcp-scan')]),
91
+ runMetaharnessAsync(['score', path]).then((r) => ['score', annotateOne(r, 'score')]),
92
+ runMetaharnessAsync(['genome', path]).then((r) => ['genome', annotateOne(r, 'genome')]),
93
+ ];
94
+ const results = await Promise.all(tasks);
95
+ return Object.fromEntries(results);
96
+ }
97
+
98
+ function persist(payload) {
99
+ const key = `audit-${new Date().toISOString().replace(/[:.]/g, '-')}`;
100
+ const r = spawnSync('npx', [
101
+ CLI_PKG, 'memory', 'store',
102
+ '--namespace', NS,
103
+ '--key', key,
104
+ '--value', JSON.stringify(payload),
105
+ ], { stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8', shell: process.platform === 'win32' });
106
+ return {
107
+ ok: r.status === 0,
108
+ namespace: NS,
109
+ key,
110
+ error: r.status === 0 ? null : (r.stderr || '').slice(0, 200),
111
+ };
112
+ }
113
+
114
+ async function main() {
115
+ if (ARGS.alertWorst !== null && !SEVERITY_RANK.hasOwnProperty(ARGS.alertWorst)) {
116
+ console.error(`oia-audit: --alert-on-worst must be one of clean|low|medium|high; got ${ARGS.alertWorst}`);
117
+ process.exit(2);
118
+ }
119
+
120
+ const startedAt = new Date().toISOString();
121
+ const wallStart = Date.now();
122
+ // iter 56 — run the 5 sub-audits in parallel (was sequential pre-iter-56).
123
+ // Worst-case wall-clock improves from 5×TIMEOUT to 1×TIMEOUT in the
124
+ // unreachable-registry case; the happy path improves from sum-of-durations
125
+ // to max-of-durations (typically ~2-4× faster).
126
+ const all = await runAllParallel(ARGS.path);
127
+ const wallMs = Date.now() - wallStart;
128
+ const { oiaManifest: oia, threatModel: tm, mcpScan: mcp, score, genome } = all;
129
+
130
+ // If all FIVE say "metaharness not available", surface the degraded
131
+ // payload exactly once and exit 0 (architectural constraint #3).
132
+ if (oia.degraded && tm.degraded && mcp.degraded && score.degraded && genome.degraded) {
133
+ emitDegradedJsonAndExit('metaharness-not-available');
134
+ return;
135
+ }
136
+
137
+ // Aggregate the worst-severity signal across mcp-scan + threat-model.
138
+ // iter 62 — safe lookup using `?? 0` so unknown severities don't
139
+ // silently bump composite via NaN-comparison (was: `acc | undefined`
140
+ // → reduce never updated). Now any [WARN]/[CRITICAL]/[ERROR] finding
141
+ // correctly elevates composite worst.
142
+ const tmWorst = String(tm.json?.worst || 'clean').toLowerCase();
143
+ const mcpFindings = Array.isArray(mcp.json?.findings) ? mcp.json.findings : [];
144
+ // iter 63 — rankSeverity() from _harness.mjs handles unknown-severity
145
+ // safely (returns 0 instead of undefined) so the reduce never sees NaN.
146
+ const mcpWorst = mcpFindings.reduce((acc, f) => {
147
+ const s = String(f.severity || 'low').toLowerCase();
148
+ return rankSeverity(s) > rankSeverity(acc) ? s : acc;
149
+ }, 'clean');
150
+ const compositeWorst = rankSeverity(tmWorst) > rankSeverity(mcpWorst) ? tmWorst : mcpWorst;
151
+
152
+ let alertTriggered = false;
153
+ let alertReason = null;
154
+ if (ARGS.alertWorst !== null) {
155
+ const threshold = rankSeverity(ARGS.alertWorst);
156
+ const compositeRank = rankSeverity(compositeWorst);
157
+ if (compositeRank >= threshold && threshold > 0) {
158
+ alertTriggered = true;
159
+ alertReason = `composite worst=${compositeWorst} ≥ ${ARGS.alertWorst}`;
160
+ }
161
+ }
162
+
163
+ // iter 59 — surface parallelization metrics. wallMs is the actual
164
+ // time the 5 subprocess race took (max of components); the sum of
165
+ // component.durationMs is what a SERIAL implementation would have
166
+ // taken. A future smoke gate compares them to catch silent
167
+ // serialization regression.
168
+ const sumComponentMs = Object.values(all).reduce(
169
+ (a, c) => a + (c?.durationMs ?? 0), 0);
170
+ const payload = {
171
+ path: ARGS.path,
172
+ startedAt,
173
+ finishedAt: new Date().toISOString(),
174
+ timing: {
175
+ wallMs,
176
+ sumComponentMs,
177
+ parallelSpeedup: sumComponentMs > 0
178
+ ? Math.round((sumComponentMs / Math.max(wallMs, 1)) * 100) / 100 : 0,
179
+ },
180
+ composite: { worst: compositeWorst, threatModelWorst: tmWorst, mcpScanWorst: mcpWorst },
181
+ components: { oiaManifest: oia, threatModel: tm, mcpScan: mcp, score, genome },
182
+ // iter 38 — denormalized harness fingerprint for cheap similarity().
183
+ // Mirrors the shape `_similarity.mjs::similarity()` expects so
184
+ // audit-trend can call it without reshuffling components.
185
+ fingerprint: {
186
+ score: score?.json && !score.degraded ? score.json : null,
187
+ genome: genome?.json && !genome.degraded ? genome.json : null,
188
+ },
189
+ alert: ARGS.alertWorst !== null ? {
190
+ threshold: ARGS.alertWorst,
191
+ triggered: alertTriggered,
192
+ reason: alertReason || `composite worst=${compositeWorst} < ${ARGS.alertWorst} — OK`,
193
+ } : null,
194
+ persisted: null,
195
+ };
196
+
197
+ if (!ARGS.dryRun) {
198
+ payload.persisted = persist(payload);
199
+ }
200
+
201
+ if (ARGS.format === 'json') {
202
+ console.log(JSON.stringify(payload, null, 2));
203
+ } else {
204
+ console.log(`# oia-audit — ${ARGS.path}`);
205
+ console.log('');
206
+ console.log(`| Component | Exit | Degraded | Duration |`);
207
+ console.log(`|---|---:|:---:|---:|`);
208
+ console.log(`| oia-manifest | ${oia.exitCode} | ${oia.degraded ? '⚠' : '✓'} | ${oia.durationMs}ms |`);
209
+ console.log(`| threat-model | ${tm.exitCode} | ${tm.degraded ? '⚠' : '✓'} | ${tm.durationMs}ms |`);
210
+ console.log(`| mcp-scan | ${mcp.exitCode} | ${mcp.degraded ? '⚠' : '✓'} | ${mcp.durationMs}ms |`);
211
+ console.log('');
212
+ console.log(`Composite worst severity: **${compositeWorst}** (tm=${tmWorst}, mcp=${mcpWorst})`);
213
+ if (payload.persisted) {
214
+ console.log(`Persisted: ${payload.persisted.ok ? `${payload.persisted.namespace}:${payload.persisted.key}` : `FAILED: ${payload.persisted.error}`}`);
215
+ }
216
+ if (payload.alert) {
217
+ console.log('');
218
+ console.log(payload.alert.triggered ? `⚠ **ALERT**: ${payload.alert.reason}` : `✓ ${payload.alert.reason}`);
219
+ }
220
+ }
221
+
222
+ if (alertTriggered) process.exit(1);
223
+ }
224
+
225
+ main().catch((e) => {
226
+ console.error('oia-audit crashed:', e?.message ?? e);
227
+ process.exit(2);
228
+ });
@@ -0,0 +1,250 @@
1
+ #!/usr/bin/env node
2
+ // router-parallel-analyze.mjs — ADR-150 Phase 2 SelfEvolvingRouter
3
+ // promotion gate.
4
+ //
5
+ // Reads paired routing decisions (Thompson bandit pick + SelfEvolvingRouter
6
+ // pick, both with measured outcomes) from a JSONL trajectory file and
7
+ // computes the THREE PROMOTION CRITERIA from ADR-150 review-round-1:
8
+ //
9
+ // (a) qualityScore improvement > 2% (relative)
10
+ // (b) usdPerDecision increase < 1% (relative)
11
+ // (c) p95 routing-decision latency increase < 5% (relative)
12
+ //
13
+ // ALL THREE must hold for promotion. AND, not OR — the OR form let
14
+ // quality gains mask cost regressions, exactly the failure mode ADR-149's
15
+ // Pareto framing was built to prevent.
16
+ //
17
+ // INPUT FORMAT (.swarm/router-parallel.jsonl, one JSON record per line):
18
+ // {
19
+ // "ts": "<iso>",
20
+ // "task": { ... },
21
+ // "bandit": { "pick": "<modelId>", "predictedQuality": 0.84, "predictedCostUsd": 0.003 },
22
+ // "ser": { "pick": "<modelId>", "predictedQuality": 0.87, "predictedCostUsd": 0.004 },
23
+ // "outcome": { "actualModel": "<modelId>", "actualQuality": 0.91, "actualUsd": 0.0035, "actualLatencyMs": 1240 }
24
+ // }
25
+ //
26
+ // USAGE
27
+ // node scripts/router-parallel-analyze.mjs --input .swarm/router-parallel.jsonl
28
+ // node scripts/router-parallel-analyze.mjs --input <file> --format json
29
+ // node scripts/router-parallel-analyze.mjs --input <file> --strict # exit 1 if NOT promotable
30
+ //
31
+ // EXIT CODES
32
+ // 0 analysis complete; in --strict mode → all three criteria passed
33
+ // 1 --strict requested AND at least one criterion failed (NOT promotable)
34
+ // 2 config error or input file missing/malformed
35
+
36
+ import { readFileSync, existsSync } from 'node:fs';
37
+
38
+ const ARGS = (() => {
39
+ const a = { input: '.swarm/router-parallel.jsonl', format: 'table', strict: false };
40
+ for (let i = 2; i < process.argv.length; i++) {
41
+ const v = process.argv[i];
42
+ if (v === '--input') a.input = process.argv[++i];
43
+ else if (v === '--format') a.format = process.argv[++i];
44
+ else if (v === '--strict') a.strict = true;
45
+ }
46
+ return a;
47
+ })();
48
+
49
+ function median(sorted) {
50
+ if (sorted.length === 0) return 0;
51
+ const n = sorted.length;
52
+ return n % 2 === 0 ? (sorted[n / 2 - 1] + sorted[n / 2]) / 2 : sorted[(n - 1) / 2];
53
+ }
54
+
55
+ function pctile(sorted, q) {
56
+ if (sorted.length === 0) return 0;
57
+ const idx = Math.floor(q * (sorted.length - 1));
58
+ return sorted[idx];
59
+ }
60
+
61
+ function mean(arr) {
62
+ if (arr.length === 0) return 0;
63
+ return arr.reduce((s, x) => s + x, 0) / arr.length;
64
+ }
65
+
66
+ function main() {
67
+ if (!existsSync(ARGS.input)) {
68
+ if (ARGS.format === 'json') {
69
+ console.log(JSON.stringify({
70
+ ok: true,
71
+ sufficient: false,
72
+ reason: 'input-file-not-found',
73
+ input: ARGS.input,
74
+ hint: 'Enable parallel logging (CLAUDE_FLOW_ROUTER_PARALLEL_LOG=1) and run a workload first.',
75
+ }, null, 2));
76
+ } else {
77
+ console.log(`# router-parallel-analyze`);
78
+ console.log('');
79
+ console.log(`_Input file not found: ${ARGS.input}_`);
80
+ console.log('');
81
+ console.log('Enable parallel logging by setting `CLAUDE_FLOW_ROUTER_PARALLEL_LOG=1`');
82
+ console.log('and running a real workload, then re-run this analyzer.');
83
+ }
84
+ process.exit(ARGS.strict ? 1 : 0);
85
+ }
86
+
87
+ let rows = [];
88
+ try {
89
+ rows = readFileSync(ARGS.input, 'utf-8')
90
+ .split('\n')
91
+ .filter((l) => l.trim())
92
+ .map((l) => JSON.parse(l));
93
+ } catch (e) {
94
+ console.error(`router-parallel-analyze: cannot parse ${ARGS.input}: ${e.message}`);
95
+ process.exit(2);
96
+ }
97
+
98
+ // Filter to records with both predictions + outcome present.
99
+ const usable = rows.filter((r) => r.bandit && r.ser && r.outcome);
100
+
101
+ if (usable.length < 30) {
102
+ const payload = {
103
+ ok: true,
104
+ sufficient: false,
105
+ reason: `n=${usable.length} < 30 (insufficient sample for AND-gate evaluation)`,
106
+ hint: 'Continue collecting parallel-routing data; statistical significance needs ≥30 paired decisions per arm.',
107
+ sampleSize: usable.length,
108
+ generatedAt: new Date().toISOString(),
109
+ };
110
+ if (ARGS.format === 'json') console.log(JSON.stringify(payload, null, 2));
111
+ else {
112
+ console.log(`# router-parallel-analyze`);
113
+ console.log('');
114
+ console.log(`Sample size: **${usable.length}** paired decisions`);
115
+ console.log('');
116
+ console.log(`_Insufficient — need ≥30 for the 3-criteria AND-gate._`);
117
+ }
118
+ process.exit(ARGS.strict ? 1 : 0);
119
+ }
120
+
121
+ // For each record, attribute the outcome to BOTH arms when they
122
+ // agreed on the model pick; when they disagreed, the actual outcome
123
+ // belongs only to the bandit (which was the executing arm). The SER
124
+ // arm gets its PREDICTED metrics for the counterfactual.
125
+ const banditOutcomes = usable.map((r) => ({
126
+ qualityActual: r.outcome.actualQuality,
127
+ usdActual: r.outcome.actualUsd,
128
+ latencyActual: r.outcome.actualLatencyMs,
129
+ }));
130
+
131
+ // SER counterfactual: use ser.predictedQuality / predictedCostUsd
132
+ // when picks disagreed; use actual when they agreed. Latency: when
133
+ // they agreed, both saw the same actual latency; when they
134
+ // disagreed, the SER pick's latency is unknown — fall back to the
135
+ // mean of agreed-decision latencies for that model.
136
+ const agreedLatencyByModel = {};
137
+ for (const r of usable) {
138
+ if (r.bandit.pick === r.ser.pick) {
139
+ const m = r.outcome.actualModel || r.bandit.pick;
140
+ (agreedLatencyByModel[m] = agreedLatencyByModel[m] || []).push(r.outcome.actualLatencyMs);
141
+ }
142
+ }
143
+ const modelMeanLatency = {};
144
+ for (const [m, arr] of Object.entries(agreedLatencyByModel)) {
145
+ modelMeanLatency[m] = mean(arr);
146
+ }
147
+
148
+ const serOutcomes = usable.map((r) => {
149
+ const agreed = r.bandit.pick === r.ser.pick;
150
+ return {
151
+ qualityActual: agreed ? r.outcome.actualQuality : r.ser.predictedQuality,
152
+ usdActual: agreed ? r.outcome.actualUsd : r.ser.predictedCostUsd,
153
+ latencyActual: agreed
154
+ ? r.outcome.actualLatencyMs
155
+ : (modelMeanLatency[r.ser.pick] ?? r.outcome.actualLatencyMs),
156
+ };
157
+ });
158
+
159
+ // Compute the three criteria.
160
+ const banditQuality = mean(banditOutcomes.map((o) => o.qualityActual));
161
+ const serQuality = mean(serOutcomes.map((o) => o.qualityActual));
162
+ const qualityImprovementPct =
163
+ banditQuality > 0 ? ((serQuality - banditQuality) / banditQuality) * 100 : 0;
164
+
165
+ const banditUsd = mean(banditOutcomes.map((o) => o.usdActual));
166
+ const serUsd = mean(serOutcomes.map((o) => o.usdActual));
167
+ const usdIncreasePct = banditUsd > 0 ? ((serUsd - banditUsd) / banditUsd) * 100 : 0;
168
+
169
+ const banditLatencyP95 = pctile(banditOutcomes.map((o) => o.latencyActual).sort((a, b) => a - b), 0.95);
170
+ const serLatencyP95 = pctile(serOutcomes.map((o) => o.latencyActual).sort((a, b) => a - b), 0.95);
171
+ const latencyIncreasePct =
172
+ banditLatencyP95 > 0 ? ((serLatencyP95 - banditLatencyP95) / banditLatencyP95) * 100 : 0;
173
+
174
+ const passes = {
175
+ quality: qualityImprovementPct > 2,
176
+ cost: usdIncreasePct < 1,
177
+ latency: latencyIncreasePct < 5,
178
+ };
179
+ const allPass = passes.quality && passes.cost && passes.latency;
180
+
181
+ // Disagreement rate (informational; not part of the gate).
182
+ const disagreements = usable.filter((r) => r.bandit.pick !== r.ser.pick).length;
183
+ const disagreementPct = (disagreements / usable.length) * 100;
184
+
185
+ const payload = {
186
+ ok: true,
187
+ sufficient: true,
188
+ sampleSize: usable.length,
189
+ disagreement: {
190
+ count: disagreements,
191
+ pct: Math.round(disagreementPct * 100) / 100,
192
+ },
193
+ bandit: {
194
+ meanQuality: Math.round(banditQuality * 10000) / 10000,
195
+ meanUsd: Math.round(banditUsd * 1e6) / 1e6,
196
+ p95LatencyMs: Math.round(banditLatencyP95),
197
+ },
198
+ ser: {
199
+ meanQuality: Math.round(serQuality * 10000) / 10000,
200
+ meanUsd: Math.round(serUsd * 1e6) / 1e6,
201
+ p95LatencyMs: Math.round(serLatencyP95),
202
+ },
203
+ criteria: {
204
+ qualityImprovementPct: Math.round(qualityImprovementPct * 100) / 100,
205
+ qualityThresholdPct: 2,
206
+ qualityPasses: passes.quality,
207
+ usdIncreasePct: Math.round(usdIncreasePct * 100) / 100,
208
+ usdThresholdPct: 1,
209
+ costPasses: passes.cost,
210
+ latencyIncreasePct: Math.round(latencyIncreasePct * 100) / 100,
211
+ latencyThresholdPct: 5,
212
+ latencyPasses: passes.latency,
213
+ },
214
+ verdict: {
215
+ promotable: allPass,
216
+ reason: allPass
217
+ ? 'All three criteria met (quality > 2% AND cost < 1% AND latency < 5%)'
218
+ : `Blocked by: ${[
219
+ !passes.quality && `quality only +${(qualityImprovementPct).toFixed(2)}% (need >2%)`,
220
+ !passes.cost && `cost +${(usdIncreasePct).toFixed(2)}% (need <1%)`,
221
+ !passes.latency && `latency +${(latencyIncreasePct).toFixed(2)}% (need <5%)`,
222
+ ].filter(Boolean).join(', ')}`,
223
+ },
224
+ generatedAt: new Date().toISOString(),
225
+ };
226
+
227
+ if (ARGS.format === 'json') {
228
+ console.log(JSON.stringify(payload, null, 2));
229
+ } else {
230
+ console.log('# router-parallel-analyze (ADR-150 SelfEvolvingRouter promotion gate)');
231
+ console.log('');
232
+ console.log(`Sample size: **${payload.sampleSize}** paired decisions`);
233
+ console.log(`Disagreement rate: ${payload.disagreement.pct}% (${payload.disagreement.count} / ${payload.sampleSize})`);
234
+ console.log('');
235
+ console.log(`| Metric | Bandit | SER | Delta | Threshold | Passes |`);
236
+ console.log(`|---|---:|---:|---:|---:|:---:|`);
237
+ console.log(`| Mean quality | ${payload.bandit.meanQuality} | ${payload.ser.meanQuality} | ${payload.criteria.qualityImprovementPct >= 0 ? '+' : ''}${payload.criteria.qualityImprovementPct}% | >+2% | ${passes.quality ? '✓' : '⚠'} |`);
238
+ console.log(`| Mean $/decision | $${payload.bandit.meanUsd.toFixed(6)} | $${payload.ser.meanUsd.toFixed(6)} | ${payload.criteria.usdIncreasePct >= 0 ? '+' : ''}${payload.criteria.usdIncreasePct}% | <+1% | ${passes.cost ? '✓' : '⚠'} |`);
239
+ console.log(`| p95 latency | ${payload.bandit.p95LatencyMs}ms | ${payload.ser.p95LatencyMs}ms | ${payload.criteria.latencyIncreasePct >= 0 ? '+' : ''}${payload.criteria.latencyIncreasePct}% | <+5% | ${passes.latency ? '✓' : '⚠'} |`);
240
+ console.log('');
241
+ console.log(`**Verdict**: ${allPass ? '✓ PROMOTABLE' : '⚠ NOT promotable'}`);
242
+ console.log('');
243
+ console.log(`_${payload.verdict.reason}_`);
244
+ }
245
+
246
+ if (ARGS.strict && !allPass) process.exit(1);
247
+ process.exit(0);
248
+ }
249
+
250
+ main();
@@ -0,0 +1,92 @@
1
+ #!/usr/bin/env node
2
+ // score.mjs — `cost-style` wrapper around `metaharness score <path>`.
3
+ //
4
+ // Returns the 5-dimension scorecard (harnessFit / compileConfidence /
5
+ // taskCoverage / toolSafety / memoryUsefulness) + estCostPerRunUsd +
6
+ // scaffoldReady boolean. Reads-only; no side effects.
7
+ //
8
+ // USAGE
9
+ // node scripts/score.mjs # current dir
10
+ // node scripts/score.mjs --path <dir> # specific dir
11
+ // node scripts/score.mjs --alert-on-fit-below 70 # exit 1 if harnessFit < 70
12
+ // node scripts/score.mjs --format json
13
+ //
14
+ // EXIT CODES
15
+ // 0 scored OK (or degraded — MetaHarness not available)
16
+ // 1 --alert-on-fit-below threshold breached
17
+ // 2 config error or scoring failure
18
+
19
+ import { runMetaharness, emitDegradedJsonAndExit } from './_harness.mjs';
20
+
21
+ const ARGS = (() => {
22
+ const a = { path: '.', format: 'json', alertFitBelow: null };
23
+ for (let i = 2; i < process.argv.length; i++) {
24
+ const v = process.argv[i];
25
+ if (v === '--path') a.path = process.argv[++i];
26
+ else if (v === '--alert-on-fit-below') a.alertFitBelow = parseFloat(process.argv[++i]);
27
+ else if (v === '--format') a.format = process.argv[++i];
28
+ }
29
+ return a;
30
+ })();
31
+
32
+ function main() {
33
+ const r = runMetaharness(['score', ARGS.path]);
34
+ if (r.degraded) {
35
+ emitDegradedJsonAndExit(r.reason);
36
+ return;
37
+ }
38
+ if (r.exitCode !== 0 || !r.json) {
39
+ console.error(`score: metaharness exited ${r.exitCode}`);
40
+ if (r.stderr) console.error(r.stderr.slice(0, 400));
41
+ process.exit(2);
42
+ }
43
+ const payload = { ...r.json, durationMs: r.durationMs };
44
+
45
+ if (ARGS.alertFitBelow !== null) {
46
+ if (!isFinite(ARGS.alertFitBelow)) {
47
+ console.error(`score: --alert-on-fit-below must be a finite number`);
48
+ process.exit(2);
49
+ }
50
+ payload.alert = {
51
+ threshold: ARGS.alertFitBelow,
52
+ triggered: typeof payload.harnessFit === 'number' && payload.harnessFit < ARGS.alertFitBelow,
53
+ reason: typeof payload.harnessFit === 'number' && payload.harnessFit < ARGS.alertFitBelow
54
+ ? `harnessFit ${payload.harnessFit} < ${ARGS.alertFitBelow}`
55
+ : `harnessFit ${payload.harnessFit ?? 'unknown'} ≥ ${ARGS.alertFitBelow} — OK`,
56
+ };
57
+ }
58
+
59
+ if (ARGS.format === 'json') {
60
+ console.log(JSON.stringify(payload, null, 2));
61
+ } else {
62
+ console.log(`# harness-score — ${payload.repo || ARGS.path}`);
63
+ console.log('');
64
+ console.log(`| Dimension | Value |`);
65
+ console.log(`|---|---:|`);
66
+ console.log(`| harnessFit | ${payload.harnessFit ?? '—'} |`);
67
+ console.log(`| compileConfidence | ${payload.compileConfidence ?? '—'} |`);
68
+ console.log(`| taskCoverage | ${payload.taskCoverage ?? '—'} |`);
69
+ console.log(`| toolSafety | ${payload.toolSafety ?? '—'} |`);
70
+ console.log(`| memoryUsefulness | ${payload.memoryUsefulness ?? '—'} |`);
71
+ console.log(`| estCostPerRunUsd | $${payload.estCostPerRunUsd ?? '—'} |`);
72
+ console.log(`| recommendedMode | ${payload.recommendedMode ?? '—'} |`);
73
+ console.log(`| archetype | ${payload.archetype ?? '—'} |`);
74
+ console.log(`| template | ${payload.template ?? '—'} |`);
75
+ console.log(`| scaffoldReady | ${payload.scaffoldReady ?? '—'} |`);
76
+ console.log(`| hardConstraints | ${payload.hardConstraints ?? '—'} |`);
77
+ console.log(`| **duration** | ${payload.durationMs}ms |`);
78
+ console.log('');
79
+ if (payload.alert) {
80
+ if (payload.alert.triggered) {
81
+ console.log(`⚠ **ALERT**: ${payload.alert.reason}`);
82
+ } else {
83
+ console.log(`✓ ${payload.alert.reason}`);
84
+ }
85
+ console.log('');
86
+ }
87
+ }
88
+
89
+ if (payload.alert?.triggered) process.exit(1);
90
+ }
91
+
92
+ main();