@claude-flow/cli 3.12.0 → 3.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -2
- package/plugins/ruflo-metaharness/.claude-flow/data/pending-insights.jsonl +1 -0
- package/plugins/ruflo-metaharness/.claude-flow/neural/stats.json +6 -0
- package/plugins/ruflo-metaharness/.claude-plugin/plugin.json +32 -0
- package/plugins/ruflo-metaharness/README.md +64 -0
- package/plugins/ruflo-metaharness/agents/metaharness-architect.md +58 -0
- package/plugins/ruflo-metaharness/commands/ruflo-metaharness.md +46 -0
- package/plugins/ruflo-metaharness/scripts/_harness.mjs +261 -0
- package/plugins/ruflo-metaharness/scripts/_similarity.mjs +161 -0
- package/plugins/ruflo-metaharness/scripts/_spike-similarity.mjs +223 -0
- package/plugins/ruflo-metaharness/scripts/audit-list.mjs +158 -0
- package/plugins/ruflo-metaharness/scripts/audit-trend.mjs +272 -0
- package/plugins/ruflo-metaharness/scripts/bench-parse-mcp-scan.mjs +146 -0
- package/plugins/ruflo-metaharness/scripts/bench-recordpair-overhead.mjs +186 -0
- package/plugins/ruflo-metaharness/scripts/bench-similarity.mjs +177 -0
- package/plugins/ruflo-metaharness/scripts/drift-from-history.mjs +363 -0
- package/plugins/ruflo-metaharness/scripts/genome.mjs +80 -0
- package/plugins/ruflo-metaharness/scripts/mcp-scan.mjs +111 -0
- package/plugins/ruflo-metaharness/scripts/mint.mjs +119 -0
- package/plugins/ruflo-metaharness/scripts/oia-audit.mjs +228 -0
- package/plugins/ruflo-metaharness/scripts/router-parallel-analyze.mjs +250 -0
- package/plugins/ruflo-metaharness/scripts/score.mjs +92 -0
- package/plugins/ruflo-metaharness/scripts/similarity.mjs +158 -0
- package/plugins/ruflo-metaharness/scripts/smoke.sh +2268 -0
- package/plugins/ruflo-metaharness/scripts/test-graceful-degradation.mjs +141 -0
- package/plugins/ruflo-metaharness/scripts/test-mcp-tools.mjs +437 -0
- package/plugins/ruflo-metaharness/scripts/test-parallel-pipeline.mjs +204 -0
- package/plugins/ruflo-metaharness/scripts/test-pipeline-roundtrip.mjs +586 -0
- package/plugins/ruflo-metaharness/scripts/test-similarity.mjs +334 -0
- package/plugins/ruflo-metaharness/scripts/test-with-openrouter.mjs +229 -0
- package/plugins/ruflo-metaharness/scripts/threat-model.mjs +59 -0
- package/plugins/ruflo-metaharness/skills/harness-drift-from-history/SKILL.md +65 -0
- package/plugins/ruflo-metaharness/skills/harness-genome/SKILL.md +54 -0
- package/plugins/ruflo-metaharness/skills/harness-mcp-scan/SKILL.md +47 -0
- package/plugins/ruflo-metaharness/skills/harness-mint/SKILL.md +72 -0
- package/plugins/ruflo-metaharness/skills/harness-oia-audit/SKILL.md +79 -0
- package/plugins/ruflo-metaharness/skills/harness-score/SKILL.md +66 -0
- package/plugins/ruflo-metaharness/skills/harness-similarity/SKILL.md +67 -0
- package/plugins/ruflo-metaharness/skills/harness-threat-model/SKILL.md +39 -0
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// _spike-similarity.mjs — ADR-152 §"Smallest demonstrable spike" gate.
|
|
3
|
+
//
|
|
4
|
+
// This file is the implementation gate for ADR-152 (Genome Similarity Search).
|
|
5
|
+
// Per ADR-151's scope-only constraint, ADR-152 cannot graduate to "Accepted"
|
|
6
|
+
// until the two invariants below are proven by running this script.
|
|
7
|
+
//
|
|
8
|
+
// THE TWO INVARIANTS (from ADR-152 §"Smallest demonstrable spike")
|
|
9
|
+
// 1. similarity(X, X) === 1 (self-match exact)
|
|
10
|
+
// 2. similarity(legal, devops) < similarity(legal, support)
|
|
11
|
+
// (vertical-affinity correct)
|
|
12
|
+
//
|
|
13
|
+
// USAGE
|
|
14
|
+
// node scripts/_spike-similarity.mjs (run + assert)
|
|
15
|
+
// node scripts/_spike-similarity.mjs --format json (CI-consumable)
|
|
16
|
+
//
|
|
17
|
+
// EXIT CODES
|
|
18
|
+
// 0 both invariants hold
|
|
19
|
+
// 1 at least one invariant fails (ADR-152 cannot graduate)
|
|
20
|
+
//
|
|
21
|
+
// LIFECYCLE
|
|
22
|
+
// Once ADR-152 is Accepted, the full similarity implementation lands in
|
|
23
|
+
// `_similarity.mjs` (production module). This file (`_spike-similarity.mjs`)
|
|
24
|
+
// remains as the regression-suite anchor for the two invariants.
|
|
25
|
+
|
|
26
|
+
const ARGS = (() => {
|
|
27
|
+
const a = { format: 'table' };
|
|
28
|
+
for (let i = 2; i < process.argv.length; i++) {
|
|
29
|
+
if (process.argv[i] === '--format') a.format = process.argv[++i];
|
|
30
|
+
}
|
|
31
|
+
return a;
|
|
32
|
+
})();
|
|
33
|
+
|
|
34
|
+
// ──────────────────────────────────────────────────────────────────────
|
|
35
|
+
// THE PROPOSED IMPLEMENTATION (ADR-152 §"Decision")
|
|
36
|
+
//
|
|
37
|
+
// Pure-TS. Takes two {genome, score} JSON shapes; emits {overall,
|
|
38
|
+
// components: {cosine, categorical, jaccard}}. The 9 numerical features +
|
|
39
|
+
// 4 categorical fields + 1 set field per ADR-152.
|
|
40
|
+
// ──────────────────────────────────────────────────────────────────────
|
|
41
|
+
|
|
42
|
+
function projectToVec(input) {
|
|
43
|
+
const s = input.score || {};
|
|
44
|
+
const g = input.genome || {};
|
|
45
|
+
return [
|
|
46
|
+
(s.harnessFit ?? 0) / 100,
|
|
47
|
+
(s.compileConfidence ?? 0) / 100,
|
|
48
|
+
(s.taskCoverage ?? 0) / 100,
|
|
49
|
+
(s.toolSafety ?? 0) / 100,
|
|
50
|
+
(s.memoryUsefulness ?? 0) / 100,
|
|
51
|
+
g.risk_score ?? 0,
|
|
52
|
+
g.test_confidence ?? 0,
|
|
53
|
+
g.publish_readiness ?? 0,
|
|
54
|
+
// estCostPerRunUsd: log10 transform, clamped to [0, 1]
|
|
55
|
+
Math.max(0, Math.min(1, Math.log10((s.estCostPerRunUsd ?? 0) + 0.001) / Math.log10(10))),
|
|
56
|
+
];
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function cosine(a, b) {
|
|
60
|
+
let dot = 0, na = 0, nb = 0;
|
|
61
|
+
for (let i = 0; i < a.length; i++) {
|
|
62
|
+
dot += a[i] * b[i];
|
|
63
|
+
na += a[i] * a[i];
|
|
64
|
+
nb += b[i] * b[i];
|
|
65
|
+
}
|
|
66
|
+
const denom = Math.sqrt(na) * Math.sqrt(nb);
|
|
67
|
+
if (denom === 0) return 0;
|
|
68
|
+
// Map cosine from [-1, 1] to [0, 1] for nonnegative vectors it's [0, 1] already
|
|
69
|
+
return Math.max(0, Math.min(1, dot / denom));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function categoricalAgreement(a, b) {
|
|
73
|
+
const fields = ['repo_type', 'archetype', 'template', 'recommendedMode'];
|
|
74
|
+
let matches = 0;
|
|
75
|
+
for (const f of fields) {
|
|
76
|
+
const av = a.genome?.[f] ?? a.score?.[f];
|
|
77
|
+
const bv = b.genome?.[f] ?? b.score?.[f];
|
|
78
|
+
if (av && bv && av === bv) matches++;
|
|
79
|
+
}
|
|
80
|
+
return matches / fields.length;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function jaccard(a, b) {
|
|
84
|
+
const A = new Set(a.genome?.agent_topology ?? []);
|
|
85
|
+
const B = new Set(b.genome?.agent_topology ?? []);
|
|
86
|
+
if (A.size === 0 && B.size === 0) return 1;
|
|
87
|
+
const intersection = [...A].filter((x) => B.has(x)).length;
|
|
88
|
+
const union = new Set([...A, ...B]).size;
|
|
89
|
+
return union === 0 ? 0 : intersection / union;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function similarity(a, b) {
|
|
93
|
+
const cos = cosine(projectToVec(a), projectToVec(b));
|
|
94
|
+
const cat = categoricalAgreement(a, b);
|
|
95
|
+
const jac = jaccard(a, b);
|
|
96
|
+
const overall = 0.6 * cos + 0.25 * cat + 0.15 * jac;
|
|
97
|
+
return {
|
|
98
|
+
overall: Math.round(overall * 10000) / 10000,
|
|
99
|
+
components: {
|
|
100
|
+
cosine: Math.round(cos * 10000) / 10000,
|
|
101
|
+
categorical: Math.round(cat * 10000) / 10000,
|
|
102
|
+
jaccard: Math.round(jac * 10000) / 10000,
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// ──────────────────────────────────────────────────────────────────────
|
|
108
|
+
// THREE SYNTHETIC FIXTURES designed to exercise the invariants. These
|
|
109
|
+
// mirror the JSON shapes `harness genome` + `harness score` emit (verified
|
|
110
|
+
// against metaharness@0.1.14 via iter-26 test-with-openrouter.mjs).
|
|
111
|
+
// ──────────────────────────────────────────────────────────────────────
|
|
112
|
+
|
|
113
|
+
const LEGAL = {
|
|
114
|
+
score: {
|
|
115
|
+
harnessFit: 78, compileConfidence: 92, taskCoverage: 65,
|
|
116
|
+
toolSafety: 88, memoryUsefulness: 70, estCostPerRunUsd: 0.04,
|
|
117
|
+
recommendedMode: 'CLI + MCP', archetype: 'compliance-harness',
|
|
118
|
+
template: 'vertical:legal', scaffoldReady: true,
|
|
119
|
+
},
|
|
120
|
+
genome: {
|
|
121
|
+
repo_type: 'node_mcp_ci',
|
|
122
|
+
agent_topology: ['contract-analyst', 'redline-reviewer', 'risk-rater', 'compliance-officer'],
|
|
123
|
+
risk_score: 0.45,
|
|
124
|
+
mcp_surface: 'remote',
|
|
125
|
+
test_confidence: 0.7,
|
|
126
|
+
publish_readiness: 0.6,
|
|
127
|
+
},
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
const SUPPORT = {
|
|
131
|
+
score: {
|
|
132
|
+
harnessFit: 75, compileConfidence: 90, taskCoverage: 70,
|
|
133
|
+
toolSafety: 90, memoryUsefulness: 72, estCostPerRunUsd: 0.05,
|
|
134
|
+
recommendedMode: 'CLI + MCP', archetype: 'compliance-harness',
|
|
135
|
+
template: 'vertical:support', scaffoldReady: true,
|
|
136
|
+
},
|
|
137
|
+
genome: {
|
|
138
|
+
repo_type: 'node_mcp_ci',
|
|
139
|
+
agent_topology: ['triager', 'kb-searcher', 'responder', 'risk-rater', 'compliance-officer'],
|
|
140
|
+
risk_score: 0.40,
|
|
141
|
+
mcp_surface: 'remote',
|
|
142
|
+
test_confidence: 0.75,
|
|
143
|
+
publish_readiness: 0.65,
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
const DEVOPS = {
|
|
148
|
+
score: {
|
|
149
|
+
harnessFit: 88, compileConfidence: 95, taskCoverage: 85,
|
|
150
|
+
toolSafety: 70, memoryUsefulness: 55, estCostPerRunUsd: 0.08,
|
|
151
|
+
recommendedMode: 'CLI', archetype: 'cli-operator-harness',
|
|
152
|
+
template: 'vertical:devops', scaffoldReady: true,
|
|
153
|
+
},
|
|
154
|
+
genome: {
|
|
155
|
+
repo_type: 'python_ops',
|
|
156
|
+
agent_topology: ['incident-commander', 'deployer', 'rollback', 'oncall-pager'],
|
|
157
|
+
risk_score: 0.25,
|
|
158
|
+
mcp_surface: 'local',
|
|
159
|
+
test_confidence: 0.85,
|
|
160
|
+
publish_readiness: 0.9,
|
|
161
|
+
},
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
// ──────────────────────────────────────────────────────────────────────
|
|
165
|
+
// INVARIANT VERIFICATION (the gate)
|
|
166
|
+
// ──────────────────────────────────────────────────────────────────────
|
|
167
|
+
|
|
168
|
+
const self = similarity(LEGAL, LEGAL);
|
|
169
|
+
const legalVsSupport = similarity(LEGAL, SUPPORT);
|
|
170
|
+
const legalVsDevops = similarity(LEGAL, DEVOPS);
|
|
171
|
+
|
|
172
|
+
const inv1Pass = self.overall === 1;
|
|
173
|
+
const inv2Pass = legalVsDevops.overall < legalVsSupport.overall;
|
|
174
|
+
|
|
175
|
+
const payload = {
|
|
176
|
+
adr: 'ADR-152',
|
|
177
|
+
spike: '_spike-similarity.mjs',
|
|
178
|
+
invariants: {
|
|
179
|
+
selfMatch: { pass: inv1Pass, value: self.overall, expected: 1 },
|
|
180
|
+
verticalAffinity: {
|
|
181
|
+
pass: inv2Pass,
|
|
182
|
+
legalVsDevops: legalVsDevops.overall,
|
|
183
|
+
legalVsSupport: legalVsSupport.overall,
|
|
184
|
+
reason: inv2Pass
|
|
185
|
+
? `legal:devops (${legalVsDevops.overall}) < legal:support (${legalVsSupport.overall}) — vertical affinity correct`
|
|
186
|
+
: `INVARIANT VIOLATED: legal:devops (${legalVsDevops.overall}) ≥ legal:support (${legalVsSupport.overall})`,
|
|
187
|
+
},
|
|
188
|
+
},
|
|
189
|
+
detail: {
|
|
190
|
+
'similarity(LEGAL, LEGAL)': self,
|
|
191
|
+
'similarity(LEGAL, SUPPORT)': legalVsSupport,
|
|
192
|
+
'similarity(LEGAL, DEVOPS)': legalVsDevops,
|
|
193
|
+
},
|
|
194
|
+
graduates: inv1Pass && inv2Pass,
|
|
195
|
+
generatedAt: new Date().toISOString(),
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
if (ARGS.format === 'json') {
|
|
199
|
+
console.log(JSON.stringify(payload, null, 2));
|
|
200
|
+
} else {
|
|
201
|
+
console.log(`# ADR-152 §Spike — Genome Similarity Search\n`);
|
|
202
|
+
console.log(`Invariant 1 — selfMatch:`);
|
|
203
|
+
console.log(` similarity(LEGAL, LEGAL).overall = ${self.overall}`);
|
|
204
|
+
console.log(` expected: 1`);
|
|
205
|
+
console.log(` ${inv1Pass ? '✓ PASS' : '✗ FAIL'}`);
|
|
206
|
+
console.log('');
|
|
207
|
+
console.log(`Invariant 2 — verticalAffinity (legal closer to support than to devops):`);
|
|
208
|
+
console.log(` similarity(LEGAL, SUPPORT).overall = ${legalVsSupport.overall}`);
|
|
209
|
+
console.log(` similarity(LEGAL, DEVOPS).overall = ${legalVsDevops.overall}`);
|
|
210
|
+
console.log(` ${inv2Pass ? '✓ PASS' : '✗ FAIL'} — ${payload.invariants.verticalAffinity.reason}`);
|
|
211
|
+
console.log('');
|
|
212
|
+
console.log(`Per-component breakdown:`);
|
|
213
|
+
console.log(` LEGAL vs SUPPORT — cosine=${legalVsSupport.components.cosine} categorical=${legalVsSupport.components.categorical} jaccard=${legalVsSupport.components.jaccard}`);
|
|
214
|
+
console.log(` LEGAL vs DEVOPS — cosine=${legalVsDevops.components.cosine} categorical=${legalVsDevops.components.categorical} jaccard=${legalVsDevops.components.jaccard}`);
|
|
215
|
+
console.log('');
|
|
216
|
+
if (payload.graduates) {
|
|
217
|
+
console.log(`✓ Both invariants hold. ADR-152 may graduate Proposed → Accepted.`);
|
|
218
|
+
} else {
|
|
219
|
+
console.log(`✗ Invariant violation. ADR-152 stays Proposed.`);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
if (!payload.graduates) process.exit(1);
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// audit-list.mjs — list timestamped oia-audit records from the
|
|
3
|
+
// `metaharness-audit` memory namespace.
|
|
4
|
+
//
|
|
5
|
+
// The iter-7 oia-audit worker writes one record per run (keyed by ISO
|
|
6
|
+
// timestamp). This script enumerates them so operators can see audit
|
|
7
|
+
// cadence, spot gaps, and pick keys to feed into audit-trend.mjs.
|
|
8
|
+
//
|
|
9
|
+
// USAGE
|
|
10
|
+
// node scripts/audit-list.mjs # most recent 20
|
|
11
|
+
// node scripts/audit-list.mjs --limit 50
|
|
12
|
+
// node scripts/audit-list.mjs --format json
|
|
13
|
+
// node scripts/audit-list.mjs --since 30d # last N days
|
|
14
|
+
//
|
|
15
|
+
// EXIT CODES
|
|
16
|
+
// 0 ok
|
|
17
|
+
// 2 config error
|
|
18
|
+
|
|
19
|
+
import { spawnSync } from 'node:child_process';
|
|
20
|
+
|
|
21
|
+
const NS = process.env.AUDIT_LIST_NAMESPACE || 'metaharness-audit';
|
|
22
|
+
const CLI_PKG = process.env.CLI_CORE === '1'
|
|
23
|
+
? '@claude-flow/cli-core@alpha'
|
|
24
|
+
: '@claude-flow/cli@latest';
|
|
25
|
+
|
|
26
|
+
const ARGS = (() => {
|
|
27
|
+
const a = { limit: 20, since: null, format: 'table' };
|
|
28
|
+
for (let i = 2; i < process.argv.length; i++) {
|
|
29
|
+
const v = process.argv[i];
|
|
30
|
+
if (v === '--limit') a.limit = parseInt(process.argv[++i], 10);
|
|
31
|
+
else if (v === '--since') a.since = process.argv[++i];
|
|
32
|
+
else if (v === '--format') a.format = process.argv[++i];
|
|
33
|
+
}
|
|
34
|
+
return a;
|
|
35
|
+
})();
|
|
36
|
+
|
|
37
|
+
function parseDurationMs(spec) {
|
|
38
|
+
const m = /^(\d+)([hdwm])$/.exec(spec);
|
|
39
|
+
if (!m) return null;
|
|
40
|
+
const n = parseInt(m[1], 10);
|
|
41
|
+
const unit = { h: 3600_000, d: 86400_000, w: 7 * 86400_000, m: 30 * 86400_000 }[m[2]];
|
|
42
|
+
return unit ? n * unit : null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function memList() {
|
|
46
|
+
const r = spawnSync('npx', [
|
|
47
|
+
CLI_PKG, 'memory', 'list',
|
|
48
|
+
'--namespace', NS, '--format', 'json',
|
|
49
|
+
], { stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8', shell: process.platform === 'win32' });
|
|
50
|
+
if (r.status !== 0) return [];
|
|
51
|
+
const m = /\[[\s\S]*\]/.exec(r.stdout || '');
|
|
52
|
+
if (!m) return [];
|
|
53
|
+
try { return JSON.parse(m[0]); } catch { return []; }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function memRetrieve(key) {
|
|
57
|
+
const r = spawnSync('npx', [
|
|
58
|
+
CLI_PKG, 'memory', 'retrieve',
|
|
59
|
+
'--namespace', NS, '--key', key,
|
|
60
|
+
], { stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8', shell: process.platform === 'win32' });
|
|
61
|
+
if (r.status !== 0) return null;
|
|
62
|
+
const m = /\{[\s\S]*\}/.exec(r.stdout || '');
|
|
63
|
+
if (!m) return null;
|
|
64
|
+
try { return JSON.parse(m[0]); } catch { return null; }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function main() {
|
|
68
|
+
if (!Number.isInteger(ARGS.limit) || ARGS.limit < 1) {
|
|
69
|
+
console.error(`audit-list: --limit must be a positive integer`);
|
|
70
|
+
process.exit(2);
|
|
71
|
+
}
|
|
72
|
+
let cutoffMs = null;
|
|
73
|
+
if (ARGS.since) {
|
|
74
|
+
const ms = parseDurationMs(ARGS.since);
|
|
75
|
+
if (!ms) {
|
|
76
|
+
console.error(`audit-list: --since must be N(h|d|w|m); got ${ARGS.since}`);
|
|
77
|
+
process.exit(2);
|
|
78
|
+
}
|
|
79
|
+
cutoffMs = Date.now() - ms;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const entries = memList()
|
|
83
|
+
.map((e) => e.key)
|
|
84
|
+
.filter((k) => typeof k === 'string' && k.startsWith('audit-'))
|
|
85
|
+
.sort()
|
|
86
|
+
.reverse(); // newest first (ISO keys sort lexicographically by time)
|
|
87
|
+
|
|
88
|
+
// Apply --since by parsing the timestamp out of the key.
|
|
89
|
+
const filtered = entries.filter((k) => {
|
|
90
|
+
if (!cutoffMs) return true;
|
|
91
|
+
// key shape: audit-2026-06-16T15-30-00-000Z
|
|
92
|
+
const m = /^audit-(.+)$/.exec(k);
|
|
93
|
+
if (!m) return true;
|
|
94
|
+
const isoLike = m[1].replace(/-/g, ':').replace(/^(\d{4}):(\d{2}):(\d{2})/, '$1-$2-$3');
|
|
95
|
+
const ts = Date.parse(isoLike);
|
|
96
|
+
return Number.isFinite(ts) && ts >= cutoffMs;
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
const slice = filtered.slice(0, ARGS.limit);
|
|
100
|
+
|
|
101
|
+
// Pull just enough fields from each record to render a compact line.
|
|
102
|
+
const rows = [];
|
|
103
|
+
for (const key of slice) {
|
|
104
|
+
const rec = memRetrieve(key);
|
|
105
|
+
if (!rec) continue;
|
|
106
|
+
rows.push({
|
|
107
|
+
key,
|
|
108
|
+
startedAt: rec.startedAt || null,
|
|
109
|
+
finishedAt: rec.finishedAt || null,
|
|
110
|
+
worst: rec.composite?.worst || null,
|
|
111
|
+
tmWorst: rec.composite?.threatModelWorst || null,
|
|
112
|
+
mcpWorst: rec.composite?.mcpScanWorst || null,
|
|
113
|
+
degraded: !!(rec.components?.oiaManifest?.degraded
|
|
114
|
+
|| rec.components?.threatModel?.degraded
|
|
115
|
+
|| rec.components?.mcpScan?.degraded),
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const payload = {
|
|
120
|
+
namespace: NS,
|
|
121
|
+
filters: { limit: ARGS.limit, since: ARGS.since },
|
|
122
|
+
totalInNamespace: entries.length,
|
|
123
|
+
matchedSinceFilter: filtered.length,
|
|
124
|
+
returned: rows.length,
|
|
125
|
+
records: rows,
|
|
126
|
+
generatedAt: new Date().toISOString(),
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
if (ARGS.format === 'json') {
|
|
130
|
+
console.log(JSON.stringify(payload, null, 2));
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
console.log(`# audit-list`);
|
|
135
|
+
console.log('');
|
|
136
|
+
console.log(`| Field | Value |`);
|
|
137
|
+
console.log(`|---|---:|`);
|
|
138
|
+
console.log(`| Namespace | \`${NS}\` |`);
|
|
139
|
+
console.log(`| Total records | ${entries.length} |`);
|
|
140
|
+
console.log(`| Matched filter | ${filtered.length} |`);
|
|
141
|
+
console.log(`| Showing | ${rows.length} (newest first) |`);
|
|
142
|
+
console.log('');
|
|
143
|
+
if (rows.length === 0) {
|
|
144
|
+
console.log(`_No audit records found. Run \`harness oia-audit\` to create one._`);
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
console.log(`| Key | Started | Composite worst | Threat | MCP | Degraded |`);
|
|
148
|
+
console.log(`|---|---|:---:|:---:|:---:|:---:|`);
|
|
149
|
+
for (const r of rows) {
|
|
150
|
+
const shortKey = r.key.slice(0, 32) + (r.key.length > 32 ? '…' : '');
|
|
151
|
+
const started = r.startedAt ? r.startedAt.slice(0, 19).replace('T', ' ') : '—';
|
|
152
|
+
console.log(`| \`${shortKey}\` | ${started} | ${r.worst || '—'} | ${r.tmWorst || '—'} | ${r.mcpWorst || '—'} | ${r.degraded ? '⚠' : '✓'} |`);
|
|
153
|
+
}
|
|
154
|
+
console.log('');
|
|
155
|
+
console.log(`_Use \`audit-trend --baseline-key <a> --current-key <b>\` to diff two records._`);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
main();
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// audit-trend.mjs — diff two oia-audit records to detect drift over time.
|
|
3
|
+
//
|
|
4
|
+
// The iter-7 oia-audit composite worker stores timestamped records under
|
|
5
|
+
// the `metaharness-audit` memory namespace. This script reads two such
|
|
6
|
+
// records and surfaces the delta:
|
|
7
|
+
// - composite worst severity change (clean → low, low → medium, etc.)
|
|
8
|
+
// - per-component (oia / threat-model / mcp-scan) status change
|
|
9
|
+
// - new HIGH-severity findings introduced
|
|
10
|
+
// - findings cleared
|
|
11
|
+
//
|
|
12
|
+
// Pairs with iter-7's oia-audit + iter-8's weekly cron — accumulated
|
|
13
|
+
// records enable drift detection without ad-hoc tooling.
|
|
14
|
+
//
|
|
15
|
+
// USAGE
|
|
16
|
+
// node scripts/audit-trend.mjs --baseline audit-<ts1>.json --current audit-<ts2>.json
|
|
17
|
+
// node scripts/audit-trend.mjs --baseline-key audit-2026-06-01... --current-key audit-2026-06-15...
|
|
18
|
+
// # pulls both from memory namespace `metaharness-audit`
|
|
19
|
+
// node scripts/audit-trend.mjs ... --alert-on-worsening
|
|
20
|
+
// node scripts/audit-trend.mjs ... --format json
|
|
21
|
+
//
|
|
22
|
+
// EXIT CODES
|
|
23
|
+
// 0 ok (no worsening, or --alert-on-worsening not set)
|
|
24
|
+
// 1 --alert-on-worsening AND composite severity worsened
|
|
25
|
+
// 2 config error or input not found
|
|
26
|
+
|
|
27
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
28
|
+
import { spawnSync } from 'node:child_process';
|
|
29
|
+
// iter 38 — structural-distance drift via ADR-152 §3.1 production module.
|
|
30
|
+
// Falls back to null if either record predates iter-38 oia-audit (no
|
|
31
|
+
// fingerprint field) — graceful degradation, never throws.
|
|
32
|
+
import { similarity } from './_similarity.mjs';
|
|
33
|
+
// iter 63 — shared SEVERITY_RANK from _harness.mjs (was a local literal
|
|
34
|
+
// missing info/warn/error/critical, which caused NaN-compare hazards).
|
|
35
|
+
import { SEVERITY_RANK, rankSeverity } from './_harness.mjs';
|
|
36
|
+
|
|
37
|
+
// iter 63 — SEVERITY_RANK moved to _harness.mjs (imported above)
|
|
38
|
+
const NS = process.env.AUDIT_TREND_NAMESPACE || 'metaharness-audit';
|
|
39
|
+
const CLI_PKG = process.env.CLI_CORE === '1'
|
|
40
|
+
? '@claude-flow/cli-core@alpha'
|
|
41
|
+
: '@claude-flow/cli@latest';
|
|
42
|
+
|
|
43
|
+
const ARGS = (() => {
|
|
44
|
+
const a = {
|
|
45
|
+
baseline: null, current: null,
|
|
46
|
+
baselineKey: null, currentKey: null,
|
|
47
|
+
alertOnWorsening: false, format: 'table',
|
|
48
|
+
// iter 38 — structural-distance gate (ADR-152 §3.1 dep)
|
|
49
|
+
alertOnDistanceBelow: null,
|
|
50
|
+
};
|
|
51
|
+
for (let i = 2; i < process.argv.length; i++) {
|
|
52
|
+
const v = process.argv[i];
|
|
53
|
+
if (v === '--baseline') a.baseline = process.argv[++i];
|
|
54
|
+
else if (v === '--current') a.current = process.argv[++i];
|
|
55
|
+
else if (v === '--baseline-key') a.baselineKey = process.argv[++i];
|
|
56
|
+
else if (v === '--current-key') a.currentKey = process.argv[++i];
|
|
57
|
+
else if (v === '--alert-on-worsening') a.alertOnWorsening = true;
|
|
58
|
+
else if (v === '--alert-on-distance-below') a.alertOnDistanceBelow = Number(process.argv[++i]);
|
|
59
|
+
else if (v === '--format') a.format = process.argv[++i];
|
|
60
|
+
}
|
|
61
|
+
return a;
|
|
62
|
+
})();
|
|
63
|
+
|
|
64
|
+
function memRetrieve(key) {
|
|
65
|
+
const r = spawnSync('npx', [
|
|
66
|
+
CLI_PKG, 'memory', 'retrieve',
|
|
67
|
+
'--namespace', NS, '--key', key,
|
|
68
|
+
], { stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8', shell: process.platform === 'win32' });
|
|
69
|
+
if (r.status !== 0) return null;
|
|
70
|
+
const m = /\{[\s\S]*\}/.exec(r.stdout || '');
|
|
71
|
+
if (!m) return null;
|
|
72
|
+
try { return JSON.parse(m[0]); } catch { return null; }
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function loadRecord(label, filePath, memKey) {
|
|
76
|
+
if (filePath) {
|
|
77
|
+
if (!existsSync(filePath)) {
|
|
78
|
+
console.error(`audit-trend: ${label} file not found: ${filePath}`);
|
|
79
|
+
process.exit(2);
|
|
80
|
+
}
|
|
81
|
+
try { return JSON.parse(readFileSync(filePath, 'utf-8')); }
|
|
82
|
+
catch (e) {
|
|
83
|
+
console.error(`audit-trend: ${label} file invalid JSON: ${e.message}`);
|
|
84
|
+
process.exit(2);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (memKey) {
|
|
88
|
+
const rec = memRetrieve(memKey);
|
|
89
|
+
if (!rec) {
|
|
90
|
+
console.error(`audit-trend: ${label} key not found in namespace ${NS}: ${memKey}`);
|
|
91
|
+
process.exit(2);
|
|
92
|
+
}
|
|
93
|
+
return rec;
|
|
94
|
+
}
|
|
95
|
+
console.error(`audit-trend: --${label} or --${label}-key is required`);
|
|
96
|
+
process.exit(2);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function main() {
|
|
100
|
+
const baseline = loadRecord('baseline', ARGS.baseline, ARGS.baselineKey);
|
|
101
|
+
const current = loadRecord('current', ARGS.current, ARGS.currentKey);
|
|
102
|
+
|
|
103
|
+
// Sanity: both must look like oia-audit records (composite + components).
|
|
104
|
+
if (!baseline.composite || !current.composite) {
|
|
105
|
+
console.error('audit-trend: snapshots must contain `composite` (from oia-audit.mjs)');
|
|
106
|
+
process.exit(2);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const baseWorst = String(baseline.composite.worst || 'clean').toLowerCase();
|
|
110
|
+
const currWorst = String(current.composite.worst || 'clean').toLowerCase();
|
|
111
|
+
// iter 63 — safe rankSeverity() eliminates NaN-compare when either
|
|
112
|
+
// severity is unknown (was a latent bug in audit-trend's drift verdict).
|
|
113
|
+
const deltaRank = rankSeverity(currWorst) - rankSeverity(baseWorst);
|
|
114
|
+
const worsened = deltaRank > 0;
|
|
115
|
+
const improved = deltaRank < 0;
|
|
116
|
+
|
|
117
|
+
// Per-component status change. Each `components.{oiaManifest, threatModel,
|
|
118
|
+
// mcpScan}` has degraded:bool + exitCode + (for threat-model) json.worst.
|
|
119
|
+
const componentDelta = (label, b, c) => {
|
|
120
|
+
if (!b || !c) return { label, delta: 'missing' };
|
|
121
|
+
if (b.degraded !== c.degraded) {
|
|
122
|
+
return { label, delta: c.degraded ? 'became degraded' : 'recovered from degraded' };
|
|
123
|
+
}
|
|
124
|
+
if (b.exitCode !== c.exitCode) {
|
|
125
|
+
return { label, delta: `exit code ${b.exitCode} → ${c.exitCode}` };
|
|
126
|
+
}
|
|
127
|
+
return { label, delta: 'unchanged' };
|
|
128
|
+
};
|
|
129
|
+
const components = {
|
|
130
|
+
oiaManifest: componentDelta('oia-manifest', baseline.components?.oiaManifest, current.components?.oiaManifest),
|
|
131
|
+
threatModel: componentDelta('threat-model', baseline.components?.threatModel, current.components?.threatModel),
|
|
132
|
+
mcpScan: componentDelta('mcp-scan', baseline.components?.mcpScan, current.components?.mcpScan),
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
// mcp-scan findings — what was introduced vs cleared.
|
|
136
|
+
const baseFindings = Array.isArray(baseline.components?.mcpScan?.json?.findings)
|
|
137
|
+
? baseline.components.mcpScan.json.findings : [];
|
|
138
|
+
const currFindings = Array.isArray(current.components?.mcpScan?.json?.findings)
|
|
139
|
+
? current.components.mcpScan.json.findings : [];
|
|
140
|
+
const fingerprint = (f) => `${f.severity}:${f.id ?? '-'}:${f.server ?? '-'}:${f.tool ?? '-'}:${(f.message ?? '').slice(0, 80)}`;
|
|
141
|
+
const baseSet = new Set(baseFindings.map(fingerprint));
|
|
142
|
+
const currSet = new Set(currFindings.map(fingerprint));
|
|
143
|
+
const introduced = currFindings.filter((f) => !baseSet.has(fingerprint(f)));
|
|
144
|
+
const cleared = baseFindings.filter((f) => !currSet.has(fingerprint(f)));
|
|
145
|
+
|
|
146
|
+
// iter 38 — structural distance via ADR-152 §3.1 similarity().
|
|
147
|
+
// Both records need a fingerprint (score+genome) — iter-38 oia-audit
|
|
148
|
+
// adds it; older records skip with verdict 'unavailable'.
|
|
149
|
+
let structuralDistance = null;
|
|
150
|
+
if (baseline.fingerprint?.score && baseline.fingerprint?.genome
|
|
151
|
+
&& current.fingerprint?.score && current.fingerprint?.genome) {
|
|
152
|
+
const sim = similarity(baseline.fingerprint, current.fingerprint);
|
|
153
|
+
structuralDistance = {
|
|
154
|
+
overall: sim.overall,
|
|
155
|
+
// Distance is the complement of similarity in [0,1]
|
|
156
|
+
distance: Number((1 - sim.overall).toFixed(4)),
|
|
157
|
+
components: sim.components,
|
|
158
|
+
verdict: sim.overall >= 0.95 ? 'near-identical'
|
|
159
|
+
: sim.overall >= 0.80 ? 'minor-drift'
|
|
160
|
+
: sim.overall >= 0.50 ? 'moderate-drift'
|
|
161
|
+
: 'major-drift',
|
|
162
|
+
};
|
|
163
|
+
} else {
|
|
164
|
+
structuralDistance = {
|
|
165
|
+
verdict: 'unavailable',
|
|
166
|
+
reason: 'one or both records predate iter-38 oia-audit fingerprint bundling',
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// Distance alert is independent of severity worsening — a harness can
|
|
171
|
+
// structurally drift while keeping the same worst-severity verdict.
|
|
172
|
+
const distanceAlertTriggered = ARGS.alertOnDistanceBelow != null
|
|
173
|
+
&& structuralDistance.overall != null
|
|
174
|
+
&& structuralDistance.overall < ARGS.alertOnDistanceBelow;
|
|
175
|
+
|
|
176
|
+
const payload = {
|
|
177
|
+
baseline: {
|
|
178
|
+
startedAt: baseline.startedAt,
|
|
179
|
+
composite: baseline.composite,
|
|
180
|
+
},
|
|
181
|
+
current: {
|
|
182
|
+
startedAt: current.startedAt,
|
|
183
|
+
composite: current.composite,
|
|
184
|
+
},
|
|
185
|
+
delta: {
|
|
186
|
+
worst: { baseline: baseWorst, current: currWorst, rankDelta: deltaRank,
|
|
187
|
+
verdict: worsened ? 'worsened' : (improved ? 'improved' : 'unchanged') },
|
|
188
|
+
components,
|
|
189
|
+
findings: {
|
|
190
|
+
introducedCount: introduced.length,
|
|
191
|
+
clearedCount: cleared.length,
|
|
192
|
+
introduced: introduced.slice(0, 20), // truncate for output sanity
|
|
193
|
+
cleared: cleared.slice(0, 20),
|
|
194
|
+
},
|
|
195
|
+
// iter 38 — structural distance via ADR-152 §3.1
|
|
196
|
+
structuralDistance,
|
|
197
|
+
},
|
|
198
|
+
alert: (ARGS.alertOnWorsening || ARGS.alertOnDistanceBelow != null) ? {
|
|
199
|
+
triggered: worsened || distanceAlertTriggered,
|
|
200
|
+
reasons: [
|
|
201
|
+
ARGS.alertOnWorsening && worsened
|
|
202
|
+
? `composite worst ${baseWorst} → ${currWorst}` : null,
|
|
203
|
+
distanceAlertTriggered
|
|
204
|
+
? `structural similarity ${structuralDistance.overall} < threshold ${ARGS.alertOnDistanceBelow}` : null,
|
|
205
|
+
].filter(Boolean),
|
|
206
|
+
} : null,
|
|
207
|
+
generatedAt: new Date().toISOString(),
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
if (ARGS.format === 'json') {
|
|
211
|
+
console.log(JSON.stringify(payload, null, 2));
|
|
212
|
+
} else {
|
|
213
|
+
console.log(`# audit-trend`);
|
|
214
|
+
console.log('');
|
|
215
|
+
console.log(`Baseline: ${baseline.startedAt}`);
|
|
216
|
+
console.log(`Current: ${current.startedAt}`);
|
|
217
|
+
console.log('');
|
|
218
|
+
console.log(`| Metric | Baseline | Current | Verdict |`);
|
|
219
|
+
console.log(`|---|---|---|---|`);
|
|
220
|
+
const verdictIcon = worsened ? '⚠ worsened' : (improved ? '✓ improved' : '— unchanged');
|
|
221
|
+
console.log(`| composite worst | ${baseWorst} | ${currWorst} | ${verdictIcon} |`);
|
|
222
|
+
for (const [label, c] of Object.entries(components)) {
|
|
223
|
+
console.log(`| ${label} | — | — | ${c.delta} |`);
|
|
224
|
+
}
|
|
225
|
+
console.log('');
|
|
226
|
+
console.log(`Findings: **+${introduced.length} introduced**, **−${cleared.length} cleared**`);
|
|
227
|
+
if (introduced.length > 0) {
|
|
228
|
+
console.log('');
|
|
229
|
+
console.log('## Introduced findings');
|
|
230
|
+
console.log('');
|
|
231
|
+
console.log('| Severity | Server | Tool | Message |');
|
|
232
|
+
console.log('|---|---|---|---|');
|
|
233
|
+
for (const f of introduced.slice(0, 20)) {
|
|
234
|
+
console.log(`| ${f.severity ?? '-'} | ${f.server ?? '-'} | ${f.tool ?? '-'} | ${(f.message ?? '').slice(0, 80)} |`);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
if (cleared.length > 0) {
|
|
238
|
+
console.log('');
|
|
239
|
+
console.log('## Cleared findings');
|
|
240
|
+
console.log('');
|
|
241
|
+
for (const f of cleared.slice(0, 5)) {
|
|
242
|
+
console.log(`- ${f.severity ?? '-'} ${f.id ?? '-'} (${f.server ?? '-'}/${f.tool ?? '-'})`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
console.log('');
|
|
246
|
+
// iter 38 — structural distance row
|
|
247
|
+
if (structuralDistance.verdict !== 'unavailable') {
|
|
248
|
+
console.log(`## Structural distance (ADR-152 §3.1)`);
|
|
249
|
+
console.log('');
|
|
250
|
+
console.log(`| Metric | Value |`);
|
|
251
|
+
console.log(`|---|---:|`);
|
|
252
|
+
console.log(`| overall similarity | ${structuralDistance.overall.toFixed(4)} |`);
|
|
253
|
+
console.log(`| distance (1 − sim) | ${structuralDistance.distance.toFixed(4)} |`);
|
|
254
|
+
console.log(`| verdict | **${structuralDistance.verdict}** |`);
|
|
255
|
+
console.log('');
|
|
256
|
+
} else {
|
|
257
|
+
console.log(`Structural distance: _unavailable — ${structuralDistance.reason}_`);
|
|
258
|
+
console.log('');
|
|
259
|
+
}
|
|
260
|
+
if (payload.alert) {
|
|
261
|
+
if (payload.alert.triggered) {
|
|
262
|
+
console.log(`⚠ **ALERT**: ${payload.alert.reasons.join('; ')}`);
|
|
263
|
+
} else {
|
|
264
|
+
console.log(`✓ no alert triggered`);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
if (payload.alert?.triggered) process.exit(1);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
main();
|