@claude-flow/cli 3.16.3 → 3.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/commands/metaharness.d.ts.map +1 -1
- package/dist/src/commands/metaharness.js +9 -3
- package/dist/src/commands/metaharness.js.map +1 -1
- package/dist/src/mcp-tools/agentdb-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/agentdb-tools.js +52 -1
- package/dist/src/mcp-tools/agentdb-tools.js.map +1 -1
- package/dist/src/mcp-tools/browser-session-tools.d.ts +5 -1
- package/dist/src/mcp-tools/browser-session-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/browser-session-tools.js +92 -18
- package/dist/src/mcp-tools/browser-session-tools.js.map +1 -1
- package/dist/src/mcp-tools/hooks-tools.d.ts +1 -0
- package/dist/src/mcp-tools/hooks-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/hooks-tools.js +131 -0
- package/dist/src/mcp-tools/hooks-tools.js.map +1 -1
- package/dist/src/mcp-tools/metaharness-tools.d.ts +7 -0
- package/dist/src/mcp-tools/metaharness-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/metaharness-tools.js +89 -0
- package/dist/src/mcp-tools/metaharness-tools.js.map +1 -1
- package/dist/src/mcp-tools/neural-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/neural-tools.js +119 -84
- package/dist/src/mcp-tools/neural-tools.js.map +1 -1
- package/dist/src/memory/memory-bridge.d.ts +15 -2
- package/dist/src/memory/memory-bridge.d.ts.map +1 -1
- package/dist/src/memory/memory-bridge.js +64 -10
- package/dist/src/memory/memory-bridge.js.map +1 -1
- package/dist/src/ruvector/output-verifier.d.ts +83 -0
- package/dist/src/ruvector/output-verifier.d.ts.map +1 -0
- package/dist/src/ruvector/output-verifier.js +277 -0
- package/dist/src/ruvector/output-verifier.js.map +1 -0
- package/dist/src/ruvector/trajectory-tree.d.ts +113 -0
- package/dist/src/ruvector/trajectory-tree.d.ts.map +1 -0
- package/dist/src/ruvector/trajectory-tree.js +237 -0
- package/dist/src/ruvector/trajectory-tree.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +5 -5
- package/plugins/ruflo-metaharness/README.md +2 -0
- package/plugins/ruflo-metaharness/agents/metaharness-architect.md +1 -1
- package/plugins/ruflo-metaharness/commands/ruflo-metaharness.md +5 -3
- package/plugins/ruflo-metaharness/scripts/_darwin.mjs +51 -41
- package/plugins/ruflo-metaharness/scripts/_harness.mjs +173 -104
- package/plugins/ruflo-metaharness/scripts/_invoke.mjs +231 -0
- package/plugins/ruflo-metaharness/scripts/_redblue.mjs +37 -70
- package/plugins/ruflo-metaharness/scripts/evolve.mjs +163 -3
- package/plugins/ruflo-metaharness/scripts/gepa.mjs +153 -0
- package/plugins/ruflo-metaharness/scripts/learn.mjs +127 -0
- package/plugins/ruflo-metaharness/scripts/smoke.sh +107 -51
- package/plugins/ruflo-metaharness/scripts/test-graceful-degradation.mjs +16 -0
- package/plugins/ruflo-metaharness/scripts/test-mcp-tools.mjs +24 -5
- package/plugins/ruflo-metaharness/skills/harness-evolve/SKILL.md +42 -3
- package/plugins/ruflo-metaharness/skills/harness-gepa/SKILL.md +65 -0
- package/plugins/ruflo-metaharness/skills/harness-learn/SKILL.md +65 -0
- package/plugins/ruflo-metaharness/skills/harness-mcp-scan/SKILL.md +3 -1
- package/plugins/ruflo-metaharness/skills/harness-score/SKILL.md +1 -1
- package/plugins/ruflo-metaharness/skills/harness-security-bench/SKILL.md +1 -1
- package/plugins/ruflo-metaharness/skills/harness-threat-model/SKILL.md +3 -1
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// gepa.mjs — surfaces the `@metaharness/darwin/gepa` LIBRARY exports.
|
|
3
|
+
//
|
|
4
|
+
// Unlike every other script in this plugin, gepa has no CLI equivalent —
|
|
5
|
+
// GEPA (darwin 0.8.0's genetic-evolution prompt-adaptation engine) ships as
|
|
6
|
+
// a library entry (`import { ... } from '@metaharness/darwin/gepa'`). This
|
|
7
|
+
// script wraps the subprocess-safe subset:
|
|
8
|
+
//
|
|
9
|
+
// genome load + validate a genome (default: the shipped cand-6 — the
|
|
10
|
+
// first holdout-confirmed cheap-tier policy promotion)
|
|
11
|
+
// validate validateGenome(json) → structural errors[]
|
|
12
|
+
// render buildSystemFromGenome(genome) → the system prompt a genome
|
|
13
|
+
// compiles to (inspect what a policy actually says)
|
|
14
|
+
// analyze analyzeTranscript(entries) → failure-class breakdown
|
|
15
|
+
//
|
|
16
|
+
// NOT SURFACED: `gepaOptimize` — it takes an in-process `evaluate(candidate)`
|
|
17
|
+
// callback ("bring your own evaluator") which cannot cross a subprocess
|
|
18
|
+
// boundary. Optimization runs belong either in library consumers
|
|
19
|
+
// (import '@metaharness/darwin/gepa' directly) or behind the darwin CLI's
|
|
20
|
+
// `evolve` verb (scripts/evolve.mjs), which pairs GEPA with its sandbox
|
|
21
|
+
// evaluators.
|
|
22
|
+
//
|
|
23
|
+
// MODULE RESOLUTION (ADR-150 graceful degradation)
|
|
24
|
+
// ================================================
|
|
25
|
+
// Delegated to _invoke.importOptionalLibrary (family-wide consolidation):
|
|
26
|
+
// 1. Try bare `import('@metaharness/darwin/gepa')` — free when the optional
|
|
27
|
+
// dep is installed in an ancestor node_modules.
|
|
28
|
+
// 2. Fall back to a ruflo-owned versioned cache install
|
|
29
|
+
// (~/.ruflo/darwin-cache-<pin>) — the versioned dir means pin bumps
|
|
30
|
+
// invalidate stale caches automatically.
|
|
31
|
+
// 3. Both fail → `{degraded: true}` exit 0. Never throws.
|
|
32
|
+
//
|
|
33
|
+
// EXIT CODES
|
|
34
|
+
// 0 op completed (or degraded)
|
|
35
|
+
// 1 --alert-on-invalid and validate found errors
|
|
36
|
+
// 2 config error (bad op / missing file)
|
|
37
|
+
|
|
38
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
39
|
+
import { importGepa, DARWIN_VERSION_PIN } from './_darwin.mjs';
|
|
40
|
+
|
|
41
|
+
// Pin lives in _darwin.mjs (DARWIN_VERSION_PIN) — single source of truth.
|
|
42
|
+
const DARWIN_PIN_VERSION = DARWIN_VERSION_PIN.split('@').pop();
|
|
43
|
+
|
|
44
|
+
const ARGS = (() => {
|
|
45
|
+
const a = {
|
|
46
|
+
op: null,
|
|
47
|
+
path: null, // genome JSON path (genome/validate/render); default cand-6
|
|
48
|
+
transcript: null, // transcript JSON path (analyze)
|
|
49
|
+
ext: undefined, // render — target file extension hint
|
|
50
|
+
glob: undefined, // render — target glob hint
|
|
51
|
+
alertOnInvalid: false,
|
|
52
|
+
format: 'json',
|
|
53
|
+
};
|
|
54
|
+
for (let i = 2; i < process.argv.length; i++) {
|
|
55
|
+
const v = process.argv[i];
|
|
56
|
+
if (v === '--op') a.op = process.argv[++i];
|
|
57
|
+
else if (v === '--path') a.path = process.argv[++i];
|
|
58
|
+
else if (v === '--transcript') a.transcript = process.argv[++i];
|
|
59
|
+
else if (v === '--ext') a.ext = process.argv[++i];
|
|
60
|
+
else if (v === '--glob') a.glob = process.argv[++i];
|
|
61
|
+
else if (v === '--alert-on-invalid') a.alertOnInvalid = true;
|
|
62
|
+
else if (v === '--format') a.format = process.argv[++i];
|
|
63
|
+
}
|
|
64
|
+
return a;
|
|
65
|
+
})();
|
|
66
|
+
|
|
67
|
+
function emitDegradedAndExit(reason) {
|
|
68
|
+
console.log(JSON.stringify({
|
|
69
|
+
degraded: true,
|
|
70
|
+
reason,
|
|
71
|
+
hint: 'Install with `npm i -D @metaharness/darwin@' + DARWIN_PIN_VERSION
|
|
72
|
+
+ '` or verify network access — the gepa entry ships inside the darwin package.',
|
|
73
|
+
generatedAt: new Date().toISOString(),
|
|
74
|
+
}, null, 2));
|
|
75
|
+
process.exit(0); // ADR-150 — ruflo stays operational without MetaHarness
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function readJsonFile(path, label) {
|
|
79
|
+
if (!path || !existsSync(path)) {
|
|
80
|
+
console.error(`gepa: ${label} file not found: ${path}`);
|
|
81
|
+
process.exit(2);
|
|
82
|
+
}
|
|
83
|
+
try {
|
|
84
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
85
|
+
} catch (e) {
|
|
86
|
+
console.error(`gepa: ${label} is not valid JSON: ${e?.message ?? e}`);
|
|
87
|
+
process.exit(2);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function loadGenomeOrExit(gepa) {
|
|
92
|
+
if (ARGS.path) {
|
|
93
|
+
if (!existsSync(ARGS.path)) {
|
|
94
|
+
console.error(`gepa: --path genome file not found: ${ARGS.path}`);
|
|
95
|
+
process.exit(2);
|
|
96
|
+
}
|
|
97
|
+
// upstream signature: loadGenome(readFileSync, path) — fs injected.
|
|
98
|
+
return { genome: gepa.loadGenome(readFileSync, ARGS.path), source: ARGS.path };
|
|
99
|
+
}
|
|
100
|
+
return { genome: gepa.loadCand6Genome(), source: gepa.CAND6_GENOME_PATH };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async function main() {
|
|
104
|
+
const OPS = ['genome', 'validate', 'render', 'analyze'];
|
|
105
|
+
if (!OPS.includes(ARGS.op)) {
|
|
106
|
+
console.error(`gepa: --op must be one of ${OPS.join('|')}`);
|
|
107
|
+
process.exit(2);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const gepa = await importGepa();
|
|
111
|
+
if (!gepa) emitDegradedAndExit('metaharness-darwin-not-available');
|
|
112
|
+
|
|
113
|
+
const start = Date.now();
|
|
114
|
+
let out;
|
|
115
|
+
|
|
116
|
+
if (ARGS.op === 'genome') {
|
|
117
|
+
const { genome, source } = loadGenomeOrExit(gepa);
|
|
118
|
+
const errors = gepa.validateGenome(genome);
|
|
119
|
+
out = { op: 'genome', source, valid: errors.length === 0, errors, genome };
|
|
120
|
+
} else if (ARGS.op === 'validate') {
|
|
121
|
+
// validate takes raw JSON (not loadGenome) so structurally-broken files
|
|
122
|
+
// reach validateGenome instead of throwing in the loader.
|
|
123
|
+
const raw = ARGS.path
|
|
124
|
+
? readJsonFile(ARGS.path, '--path genome')
|
|
125
|
+
: gepa.loadCand6Genome();
|
|
126
|
+
const errors = gepa.validateGenome(raw);
|
|
127
|
+
out = { op: 'validate', source: ARGS.path ?? gepa.CAND6_GENOME_PATH, valid: errors.length === 0, errors };
|
|
128
|
+
} else if (ARGS.op === 'render') {
|
|
129
|
+
const { genome, source } = loadGenomeOrExit(gepa);
|
|
130
|
+
const system = gepa.buildSystemFromGenome(genome, ARGS.ext, ARGS.glob);
|
|
131
|
+
out = { op: 'render', source, chars: system.length, system };
|
|
132
|
+
} else {
|
|
133
|
+
// analyze
|
|
134
|
+
const entries = readJsonFile(ARGS.transcript, '--transcript');
|
|
135
|
+
if (!Array.isArray(entries)) {
|
|
136
|
+
console.error('gepa: --transcript must be a JSON array of transcript entries');
|
|
137
|
+
process.exit(2);
|
|
138
|
+
}
|
|
139
|
+
const analysis = gepa.analyzeTranscript(entries);
|
|
140
|
+
out = { op: 'analyze', source: ARGS.transcript, entries: entries.length, analysis };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
out.durationMs = Date.now() - start;
|
|
144
|
+
console.log(JSON.stringify(out, null, 2));
|
|
145
|
+
|
|
146
|
+
if (ARGS.alertOnInvalid && out.valid === false) process.exit(1);
|
|
147
|
+
process.exit(0);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
main().catch((e) => {
|
|
151
|
+
console.error(`gepa: ${e?.message ?? e}`);
|
|
152
|
+
process.exit(2);
|
|
153
|
+
});
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// learn.mjs — wrapper around `metaharness learn` (upstream ADR-235, metaharness@0.3.0).
|
|
3
|
+
//
|
|
4
|
+
// GEPA learning run: optimizes a harness genome against a SWE-bench-style
|
|
5
|
+
// slice manifest. $0 DRY-RUN BY DEFAULT — upstream only spends (model calls,
|
|
6
|
+
// Docker sandboxes) when --run is passed, and we forward that flag verbatim
|
|
7
|
+
// so the spend opt-in stays explicit at every layer.
|
|
8
|
+
//
|
|
9
|
+
// CHECKOUT PRECONDITION (upstream design, not a bug)
|
|
10
|
+
// ==================================================
|
|
11
|
+
// The learning harness (GEPA + SWE-bench + Docker) is too heavy to ship in
|
|
12
|
+
// the npm package, so `metaharness learn` requires a local clone of the
|
|
13
|
+
// metaharness repo, located via $METAHARNESS_REPO or by running inside the
|
|
14
|
+
// clone. When the checkout is absent we emit a structured
|
|
15
|
+
// `{status: "checkout-required"}` payload and exit 0 — the script ran as
|
|
16
|
+
// designed and told the agent what to do next. This is distinct from
|
|
17
|
+
// `degraded: true` (npm package absent). The managed-service path (gateway-
|
|
18
|
+
// side learn jobs, no checkout) is upstream's ADR-235 follow-up.
|
|
19
|
+
//
|
|
20
|
+
// USAGE
|
|
21
|
+
// node scripts/learn.mjs --host claude-code --model haiku --slice slices/lite.json
|
|
22
|
+
// node scripts/learn.mjs --repo ~/src/metaharness --host codex --model gpt-5-mini --slice s.json --run
|
|
23
|
+
//
|
|
24
|
+
// EXIT CODES
|
|
25
|
+
// 0 learn completed (or dry-run report, or degraded, or checkout-required)
|
|
26
|
+
// 1 --alert-on-fail and the learn run reported failure
|
|
27
|
+
// 2 config error (bad arg)
|
|
28
|
+
|
|
29
|
+
import { existsSync } from 'node:fs';
|
|
30
|
+
import { runMetaharnessAsync, emitDegradedJsonAndExit } from './_harness.mjs';
|
|
31
|
+
|
|
32
|
+
const ARGS = (() => {
|
|
33
|
+
const a = {
|
|
34
|
+
host: null,
|
|
35
|
+
model: null,
|
|
36
|
+
slice: null,
|
|
37
|
+
repo: null,
|
|
38
|
+
run: false,
|
|
39
|
+
alertOnFail: false,
|
|
40
|
+
format: 'json',
|
|
41
|
+
timeoutMs: null,
|
|
42
|
+
};
|
|
43
|
+
for (let i = 2; i < process.argv.length; i++) {
|
|
44
|
+
const v = process.argv[i];
|
|
45
|
+
if (v === '--host') a.host = process.argv[++i];
|
|
46
|
+
else if (v === '--model') a.model = process.argv[++i];
|
|
47
|
+
else if (v === '--slice') a.slice = process.argv[++i];
|
|
48
|
+
else if (v === '--repo') a.repo = process.argv[++i];
|
|
49
|
+
else if (v === '--run') a.run = true;
|
|
50
|
+
else if (v === '--alert-on-fail') a.alertOnFail = true;
|
|
51
|
+
else if (v === '--format') a.format = process.argv[++i];
|
|
52
|
+
else if (v === '--timeout-ms') a.timeoutMs = parseInt(process.argv[++i], 10);
|
|
53
|
+
}
|
|
54
|
+
return a;
|
|
55
|
+
})();
|
|
56
|
+
|
|
57
|
+
const CHECKOUT_RX = /requires a metaharness repo checkout/i;
|
|
58
|
+
|
|
59
|
+
function defaultTimeoutMs() {
|
|
60
|
+
// Dry-run resolves the slice manifest + prices the run without spending —
|
|
61
|
+
// bounded by repo scan, not model calls. Real runs (--run) are dominated
|
|
62
|
+
// by model calls × slice size; callers should pass --timeout-ms matched
|
|
63
|
+
// to their slice. 10 min is a floor for small slices, not a budget.
|
|
64
|
+
return ARGS.run ? 600_000 : 120_000;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function main() {
|
|
68
|
+
if (ARGS.repo && !existsSync(ARGS.repo)) {
|
|
69
|
+
console.error(`learn: --repo path does not exist: ${ARGS.repo}`);
|
|
70
|
+
process.exit(2);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const cliArgs = ['learn'];
|
|
74
|
+
if (ARGS.host) cliArgs.push('--host', ARGS.host);
|
|
75
|
+
if (ARGS.model) cliArgs.push('--model', ARGS.model);
|
|
76
|
+
if (ARGS.slice) cliArgs.push('--slice', ARGS.slice);
|
|
77
|
+
if (ARGS.run) cliArgs.push('--run');
|
|
78
|
+
|
|
79
|
+
const r = await runMetaharnessAsync(cliArgs, {
|
|
80
|
+
// learn emits a human-readable report; --json support is not guaranteed
|
|
81
|
+
// across 0.3.x, so we don't inject it and parse leniently instead.
|
|
82
|
+
json: false,
|
|
83
|
+
timeoutMs: ARGS.timeoutMs ?? defaultTimeoutMs(),
|
|
84
|
+
env: ARGS.repo ? { METAHARNESS_REPO: ARGS.repo } : {},
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
if (r.degraded) emitDegradedJsonAndExit(r.reason);
|
|
88
|
+
|
|
89
|
+
const combined = `${r.stdout}\n${r.stderr}`;
|
|
90
|
+
if (CHECKOUT_RX.test(combined)) {
|
|
91
|
+
console.log(JSON.stringify({
|
|
92
|
+
status: 'checkout-required',
|
|
93
|
+
hint: 'git clone https://github.com/ruvnet/metaharness.git, then re-run with '
|
|
94
|
+
+ '--repo /path/to/metaharness (or set METAHARNESS_REPO). The learning '
|
|
95
|
+
+ 'harness (GEPA + SWE-bench + Docker) is not shipped in the npm package.',
|
|
96
|
+
dryRun: !ARGS.run,
|
|
97
|
+
durationMs: r.durationMs,
|
|
98
|
+
}, null, 2));
|
|
99
|
+
process.exit(0);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Lenient JSON extraction — grab the last {...} block if one exists.
|
|
103
|
+
let json = null;
|
|
104
|
+
const matches = [...r.stdout.matchAll(/\{[\s\S]*?\}/g)];
|
|
105
|
+
for (let i = matches.length - 1; i >= 0; i--) {
|
|
106
|
+
try { json = JSON.parse(matches[i][0]); break; } catch { /* try previous */ }
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const out = {
|
|
110
|
+
status: r.exitCode === 0 ? 'ok' : 'failed',
|
|
111
|
+
dryRun: !ARGS.run,
|
|
112
|
+
exitCode: r.exitCode,
|
|
113
|
+
report: json,
|
|
114
|
+
// When upstream emits no JSON, the raw report is still the deliverable.
|
|
115
|
+
rawReport: json ? undefined : r.stdout.slice(0, 20_000),
|
|
116
|
+
durationMs: r.durationMs,
|
|
117
|
+
};
|
|
118
|
+
console.log(JSON.stringify(out, null, 2));
|
|
119
|
+
|
|
120
|
+
if (ARGS.alertOnFail && r.exitCode !== 0) process.exit(1);
|
|
121
|
+
process.exit(0);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
main().catch((e) => {
|
|
125
|
+
console.error(`learn: ${e?.message ?? e}`);
|
|
126
|
+
process.exit(2);
|
|
127
|
+
});
|
|
@@ -8,6 +8,29 @@ step() { printf "→ %s ... " "$1"; }
|
|
|
8
8
|
ok() { printf "PASS\n"; PASS=$((PASS+1)); }
|
|
9
9
|
bad() { printf "FAIL: %s\n" "$1"; FAIL=$((FAIL+1)); }
|
|
10
10
|
|
|
11
|
+
# ---------------------------------------------------------------------------
|
|
12
|
+
# Derived surface counts (converged arch review, 2026-07).
|
|
13
|
+
# The MCP-tool count (was hardcoded 15 in 4 places) and the CLI-subcommand
|
|
14
|
+
# count (was hardcoded 13 in 2 places, plus the 16 footnote) are derived ONCE
|
|
15
|
+
# from the source-of-truth files here. Every assertion below compares an
|
|
16
|
+
# INDEPENDENT surface (CLAUDE.md catalog, test-mcp-tools literal, footnote
|
|
17
|
+
# occurrences, runScript refs) against these derived values — never a surface
|
|
18
|
+
# against itself. Adding a tool/subcommand upstream now only requires updating
|
|
19
|
+
# the OTHER surfaces, not this script.
|
|
20
|
+
TOOLS_SRC="$ROOT/../../v3/@claude-flow/cli/src/mcp-tools/metaharness-tools.ts"
|
|
21
|
+
SUBS_SRC="$ROOT/../../v3/@claude-flow/cli/src/commands/metaharness.ts"
|
|
22
|
+
EXPECTED_TOOLS=$(grep -cE "name: 'metaharness_" "$TOOLS_SRC" 2>/dev/null; true)
|
|
23
|
+
EXPECTED_SUBS=$(grep -cE "^[[:space:]]+'?[a-z-]+'?:[[:space:]]*'[a-z-]+\.mjs'" "$SUBS_SRC" 2>/dev/null; true)
|
|
24
|
+
: "${EXPECTED_TOOLS:=0}"
|
|
25
|
+
: "${EXPECTED_SUBS:=0}"
|
|
26
|
+
|
|
27
|
+
step "0. derived surface counts extracted (tools >= 15, subcommands >= 13 — regex-rot floor)"
|
|
28
|
+
if [[ "$EXPECTED_TOOLS" -ge 15 && "$EXPECTED_SUBS" -ge 13 ]]; then
|
|
29
|
+
ok
|
|
30
|
+
else
|
|
31
|
+
bad "extraction-regex-rot: EXPECTED_TOOLS=$EXPECTED_TOOLS EXPECTED_SUBS=$EXPECTED_SUBS"
|
|
32
|
+
fi
|
|
33
|
+
|
|
11
34
|
step "1. plugin.json declares 0.1.0 with adr-150 keywords"
|
|
12
35
|
v=$(grep -E '"version"' "$ROOT/.claude-plugin/plugin.json" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
|
|
13
36
|
if [[ "$v" != "0.1.0" ]]; then
|
|
@@ -45,6 +68,26 @@ grep -q "metaharness-not-available" "$F" || miss="$miss no-degraded-reason"
|
|
|
45
68
|
grep -q "degraded: true" "$F" || miss="$miss no-degraded-flag"
|
|
46
69
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
47
70
|
|
|
71
|
+
step "3b. _invoke.mjs shared plumbing layer (consolidation, 2026-07)"
|
|
72
|
+
F="$ROOT/scripts/_invoke.mjs"
|
|
73
|
+
miss=""
|
|
74
|
+
[[ -f "$F" ]] || miss="$miss missing"
|
|
75
|
+
node --check "$F" 2>/dev/null || miss="$miss syntax-error"
|
|
76
|
+
# The consolidated helpers every adapter (_harness/_darwin/_redblue/gepa) uses
|
|
77
|
+
for sym in DEGRADED_RX classifyDegraded injectJson parseTrailingJson ensureCachedInstall makeDegradedEmitter importOptionalLibrary findLocalPackageDir; do
|
|
78
|
+
grep -q "export.*${sym}" "$F" || miss="$miss no-${sym}"
|
|
79
|
+
done
|
|
80
|
+
# Superset degraded regex must include the npm-level failure marker
|
|
81
|
+
grep -q "npm ERR" "$F" || miss="$miss degraded-rx-missing-npm-err"
|
|
82
|
+
# The -timeout vs -not-available distinction is load-bearing
|
|
83
|
+
grep -q -- "-timeout" "$F" || miss="$miss no-timeout-reason"
|
|
84
|
+
grep -q -- "-not-available" "$F" || miss="$miss no-not-available-reason"
|
|
85
|
+
# All four adapters import from it
|
|
86
|
+
for a in _harness _darwin _redblue gepa; do
|
|
87
|
+
grep -q "from './_invoke.mjs'\|from './_darwin.mjs'" "$ROOT/scripts/${a}.mjs" || miss="$miss ${a}-not-adapting"
|
|
88
|
+
done
|
|
89
|
+
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
90
|
+
|
|
48
91
|
step "4. score.mjs harness present + parses + uses _harness.mjs + alert"
|
|
49
92
|
F="$ROOT/scripts/score.mjs"
|
|
50
93
|
miss=""
|
|
@@ -151,9 +194,9 @@ F="$ROOT/../../v3/@claude-flow/cli/src/commands/metaharness.ts"
|
|
|
151
194
|
miss=""
|
|
152
195
|
[[ -f "$F" ]] || miss="$miss command-file-missing"
|
|
153
196
|
grep -q "name: 'metaharness'" "$F" 2>/dev/null || miss="$miss no-name-field"
|
|
154
|
-
# All
|
|
197
|
+
# All core subcommands must each be present in the dispatch table.
|
|
155
198
|
# Match either quoted ('mcp-scan': ...) or unquoted shorthand (score: ...) keys.
|
|
156
|
-
for sub in score genome mcp-scan threat-model oia-audit audit-list audit-trend mint redblue; do
|
|
199
|
+
for sub in score genome mcp-scan threat-model oia-audit audit-list audit-trend mint redblue learn gepa; do
|
|
157
200
|
grep -qE "(^|[[:space:]])'?${sub}'?:" "$F" 2>/dev/null || miss="$miss missing-$sub"
|
|
158
201
|
done
|
|
159
202
|
# Registered in the loader
|
|
@@ -170,24 +213,29 @@ if (!od.metaharness) { console.error('missing metaharness in optionalDependencie
|
|
|
170
213
|
if (j.dependencies && j.dependencies.metaharness) { console.error('metaharness leaked into dependencies'); process.exit(1); }
|
|
171
214
|
" 2>/dev/null && ok || bad "ruflo wrapper missing metaharness optionalDep"
|
|
172
215
|
|
|
173
|
-
step "17r. _harness.mjs
|
|
216
|
+
step "17r. _harness.mjs pinned-version + no-@latest regression guard (supersedes iter 27)"
|
|
174
217
|
F="$ROOT/scripts/_harness.mjs"
|
|
175
218
|
miss=""
|
|
176
|
-
#
|
|
177
|
-
# to
|
|
178
|
-
#
|
|
179
|
-
#
|
|
180
|
-
#
|
|
181
|
-
#
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
if grep -qE "execCli\(\s*['\"]-y metaharness@latest['\"]" "$F" 2>/dev/null; then
|
|
185
|
-
miss="$miss bug-regressed-string-form"
|
|
219
|
+
# SECURITY (HIGH, converged review 2026-07): the pre-consolidation helper
|
|
220
|
+
# shelled to `npx -y metaharness@latest` — a compromised upstream publish
|
|
221
|
+
# would execute arbitrary code on user machines, and @latest forced a
|
|
222
|
+
# registry check per call. Lock the fix: NO @latest anywhere in the loader,
|
|
223
|
+
# a pinned tilde range, and node-direct invocation of resolved bin paths
|
|
224
|
+
# (local walk-up install or one-time versioned cache).
|
|
225
|
+
if grep -q "metaharness@latest" "$F" 2>/dev/null; then
|
|
226
|
+
miss="$miss at-latest-regressed"
|
|
186
227
|
fi
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
228
|
+
grep -q "METAHARNESS_PIN_VERSION = '~" "$F" 2>/dev/null || miss="$miss no-pinned-range"
|
|
229
|
+
# node-direct spawn of the resolved bin (not an npx shim)
|
|
230
|
+
grep -qE "spawnSync\('node'" "$F" 2>/dev/null || miss="$miss no-node-direct-sync"
|
|
231
|
+
grep -qE "spawn\('node'" "$F" 2>/dev/null || miss="$miss no-node-direct-async"
|
|
232
|
+
# resolution order: local install first, then versioned cache install
|
|
233
|
+
grep -q "findLocalPackageDir" "$F" 2>/dev/null || miss="$miss no-local-resolution"
|
|
234
|
+
grep -q "ensureCachedInstall" "$F" 2>/dev/null || miss="$miss no-cache-install"
|
|
235
|
+
# BOTH bins resolved from the package.json bin map (metaharness + harness)
|
|
236
|
+
grep -q "bin.metaharness" "$F" 2>/dev/null || miss="$miss no-metaharness-bin"
|
|
237
|
+
grep -q "bin.harness" "$F" 2>/dev/null || miss="$miss no-harness-bin"
|
|
238
|
+
# cwd + env pass-through (iter 27 behavior retained)
|
|
191
239
|
grep -q "cwd: opts" "$F" || miss="$miss no-cwd-passthrough"
|
|
192
240
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
193
241
|
|
|
@@ -722,7 +770,7 @@ for k in $KEYS; do
|
|
|
722
770
|
grep -qE "npx ruflo metaharness ${k}([ \\\\]|$)" "$CMD" 2>/dev/null \
|
|
723
771
|
|| miss="$miss subcommand-${k}-not-in-claude-md"
|
|
724
772
|
done
|
|
725
|
-
[[ "$COUNT" == "
|
|
773
|
+
[[ "$COUNT" == "$EXPECTED_SUBS" ]] || miss="$miss subcommand-count-stale:$COUNT-expected-$EXPECTED_SUBS"
|
|
726
774
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
727
775
|
|
|
728
776
|
step "17z56. every MCP tool documented in CLAUDE.md (iter 93)"
|
|
@@ -742,11 +790,12 @@ for t in $TOOLS; do
|
|
|
742
790
|
grep -q "mcp__claude-flow__${t}" "$CMD" 2>/dev/null \
|
|
743
791
|
|| miss="$miss ${t}-not-in-claude-md"
|
|
744
792
|
done
|
|
745
|
-
#
|
|
746
|
-
#
|
|
747
|
-
#
|
|
748
|
-
#
|
|
749
|
-
|
|
793
|
+
# Count derived from the wrapper source (mint deliberately excluded — see
|
|
794
|
+
# iter 73). The historical bump ladder (9→12→13→15) is gone: the extraction
|
|
795
|
+
# loop above and $EXPECTED_TOOLS both come from metaharness-tools.ts, so this
|
|
796
|
+
# equality only catches loop/derivation divergence; the REAL cross-surface
|
|
797
|
+
# check is the per-tool CLAUDE.md grep in the loop.
|
|
798
|
+
[[ "$COUNT" == "$EXPECTED_TOOLS" ]] || miss="$miss mcp-tool-count-stale:$COUNT-expected-$EXPECTED_TOOLS"
|
|
750
799
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
751
800
|
|
|
752
801
|
step "17z55. MCP enum + SEVERITY_RANK vocabulary aligned (iter 92)"
|
|
@@ -821,10 +870,11 @@ for f in $REFS; do
|
|
|
821
870
|
COUNT=$((COUNT + 1))
|
|
822
871
|
[[ -f "$SCRIPTS_DIR/$f" ]] || miss="$miss mcp-script-${f}-missing"
|
|
823
872
|
done
|
|
824
|
-
#
|
|
825
|
-
#
|
|
826
|
-
#
|
|
827
|
-
|
|
873
|
+
# One unique runScript() ref per MCP tool (mint deliberately excluded).
|
|
874
|
+
# Cross-aspect check: runScript('*.mjs') call sites vs the derived
|
|
875
|
+
# `name: 'metaharness_*'` declaration count in the same file — a handler
|
|
876
|
+
# added without a script ref (or vice-versa) diverges these.
|
|
877
|
+
[[ "$COUNT" == "$EXPECTED_TOOLS" ]] || miss="$miss mcp-script-count-stale:$COUNT-expected-$EXPECTED_TOOLS"
|
|
828
878
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
829
879
|
|
|
830
880
|
step "17z52. SUBCOMMANDS map entries point at existing script files (iter 89)"
|
|
@@ -844,9 +894,10 @@ for f in $MAPPINGS; do
|
|
|
844
894
|
COUNT=$((COUNT + 1))
|
|
845
895
|
[[ -f "$SCRIPTS_DIR/$f" ]] || miss="$miss script-${f}-missing"
|
|
846
896
|
done
|
|
847
|
-
#
|
|
848
|
-
#
|
|
849
|
-
|
|
897
|
+
# SUBCOMMANDS map entry count vs the derived $EXPECTED_SUBS (same source,
|
|
898
|
+
# so this catches loop/derivation divergence; the real check is the per-file
|
|
899
|
+
# existence assertion in the loop above).
|
|
900
|
+
[[ "$COUNT" == "$EXPECTED_SUBS" ]] || miss="$miss mapping-count-stale:$COUNT-expected-$EXPECTED_SUBS"
|
|
850
901
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
851
902
|
|
|
852
903
|
step "17z51. all metaharness scripts produce parseable JSON in --format json mode (iter 88)"
|
|
@@ -1140,9 +1191,9 @@ step "17z36. CLI subcommand list current + mint anti-MCP guard (iter 73)"
|
|
|
1140
1191
|
miss=""
|
|
1141
1192
|
DISP="$ROOT/../../v3/@claude-flow/cli/src/commands/metaharness.ts"
|
|
1142
1193
|
WRAPPER="$ROOT/../../v3/@claude-flow/cli/src/mcp-tools/metaharness-tools.ts"
|
|
1143
|
-
# All
|
|
1144
|
-
#
|
|
1145
|
-
for sub in score genome mcp-scan threat-model oia-audit audit-list audit-trend similarity drift-from-history mint redblue; do
|
|
1194
|
+
# All 13 dispatchable subcommands present (iter-73 had 10; @metaharness/redblue
|
|
1195
|
+
# added redblue.mjs; metaharness@0.3.0/darwin@0.8.0 added learn.mjs + gepa.mjs).
|
|
1196
|
+
for sub in score genome mcp-scan threat-model oia-audit audit-list audit-trend similarity drift-from-history mint redblue learn gepa; do
|
|
1146
1197
|
grep -q "${sub}" "$DISP" 2>/dev/null || miss="$miss subcommand-${sub}-not-listed"
|
|
1147
1198
|
done
|
|
1148
1199
|
# ANTI-MINT GUARD: mint is intentionally CLI-only per ADR-150 §Sandboxing
|
|
@@ -1399,10 +1450,13 @@ grep -q "iter 58 — reuse auditResult from the parallel batch" "$F" 2>/dev/null
|
|
|
1399
1450
|
|
|
1400
1451
|
step "17z20. iter-55 gaps B + C closed (iter 57)"
|
|
1401
1452
|
miss=""
|
|
1402
|
-
# Gap B:
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1453
|
+
# Gap B: the degraded-classification regex catches ENOTFOUND-class network
|
|
1454
|
+
# errors. Consolidation (2026-07) moved DEGRADED_RX to _invoke.mjs — check it
|
|
1455
|
+
# there, plus confirm _harness.mjs actually routes through the shared classifier.
|
|
1456
|
+
INVOKE="$ROOT/scripts/_invoke.mjs"
|
|
1457
|
+
grep -q "ENOTFOUND" "$INVOKE" 2>/dev/null || miss="$miss no-enotfound-regex"
|
|
1458
|
+
grep -q "getaddrinfo\|ECONNREFUSED\|ETIMEDOUT" "$INVOKE" 2>/dev/null || miss="$miss no-network-error-regex"
|
|
1459
|
+
grep -q "classifyDegraded" "$ROOT/scripts/_harness.mjs" 2>/dev/null || miss="$miss harness-not-using-shared-classifier"
|
|
1406
1460
|
# Gap C: drift-from-history probes oia-audit to disambiguate no-history vs dep-absent
|
|
1407
1461
|
DRIFT="$ROOT/scripts/drift-from-history.mjs"
|
|
1408
1462
|
grep -q "disambiguate" "$DRIFT" 2>/dev/null || miss="$miss no-disambiguate-comment"
|
|
@@ -1628,12 +1682,11 @@ CODE=$?
|
|
|
1628
1682
|
step "17z9. MCP success-semantic footnote + audit_trend file inputs (iter 46)"
|
|
1629
1683
|
miss=""
|
|
1630
1684
|
WRAPPER="$ROOT/../../v3/@claude-flow/cli/src/mcp-tools/metaharness-tools.ts"
|
|
1631
|
-
# Success-semantic constant declared + appended to N descriptions =
|
|
1632
|
-
#
|
|
1633
|
-
#
|
|
1634
|
-
# @metaharness/redblue integration added 1 more → 1 + 13 = 14.
|
|
1685
|
+
# Success-semantic constant declared + appended to N tool descriptions =
|
|
1686
|
+
# N+1 occurrences. N is the derived tool count — cross-aspect check within
|
|
1687
|
+
# the wrapper: footnote appends vs `name:` declarations.
|
|
1635
1688
|
COUNT=$(grep -c "MCP_SUCCESS_SEMANTIC" "$WRAPPER" 2>/dev/null; true)
|
|
1636
|
-
[[ "$COUNT" == "
|
|
1689
|
+
[[ "$COUNT" == "$((EXPECTED_TOOLS + 1))" ]] || miss="$miss footnote-count:$COUNT-expected-$((EXPECTED_TOOLS + 1))"
|
|
1637
1690
|
# audit_trend now exposes baselineFile / currentFile
|
|
1638
1691
|
grep -q "baselineFile" "$WRAPPER" 2>/dev/null || miss="$miss no-baseline-file"
|
|
1639
1692
|
grep -q "currentFile" "$WRAPPER" 2>/dev/null || miss="$miss no-current-file"
|
|
@@ -1671,10 +1724,9 @@ grep -q "success = exitCode === 0" "$WRAPPER" 2>/dev/null || miss="$miss no-exit
|
|
|
1671
1724
|
COUNT_OLD=$(grep -c "success: !r.degraded" "$WRAPPER" 2>/dev/null; true)
|
|
1672
1725
|
[[ "$COUNT_OLD" == "0" ]] || miss="$miss old-pattern-still-present:$COUNT_OLD"
|
|
1673
1726
|
COUNT_NEW=$(grep -c "success: r.success" "$WRAPPER" 2>/dev/null; true)
|
|
1674
|
-
#
|
|
1675
|
-
#
|
|
1676
|
-
|
|
1677
|
-
[[ "$COUNT_NEW" == "13" ]] || miss="$miss new-pattern-count:$COUNT_NEW-expected-13"
|
|
1727
|
+
# One `success: r.success` per handler = the derived tool count. Cross-aspect
|
|
1728
|
+
# check within the wrapper: handler success-wiring vs `name:` declarations.
|
|
1729
|
+
[[ "$COUNT_NEW" == "$EXPECTED_TOOLS" ]] || miss="$miss new-pattern-count:$COUNT_NEW-expected-$EXPECTED_TOOLS"
|
|
1678
1730
|
# Runtime anchors: iter 44 success assertions present
|
|
1679
1731
|
T="$ROOT/scripts/test-mcp-tools.mjs"
|
|
1680
1732
|
grep -q "iter 44 fix" "$T" 2>/dev/null || miss="$miss no-iter44-anchors"
|
|
@@ -2032,13 +2084,17 @@ grep -q "result has 'success'" "$F" || miss="$miss no-success-assertion"
|
|
|
2032
2084
|
grep -q "result has 'data'" "$F" || miss="$miss no-data-assertion"
|
|
2033
2085
|
grep -q "result has 'degraded'" "$F" || miss="$miss no-degraded-assertion"
|
|
2034
2086
|
grep -q "result has 'exitCode'" "$F" || miss="$miss no-exitcode-assertion"
|
|
2035
|
-
# All
|
|
2036
|
-
# ADR-153 added bench/evolve/security_bench, @metaharness/redblue added redblue
|
|
2037
|
-
|
|
2087
|
+
# All 15 tool names enumerated (similarity iter 36, drift_from_history iter 54,
|
|
2088
|
+
# ADR-153 added bench/evolve/security_bench, @metaharness/redblue added redblue,
|
|
2089
|
+
# metaharness@0.3.0/darwin@0.8.0 added learn + gepa)
|
|
2090
|
+
for tool in metaharness_score metaharness_genome metaharness_mcp_scan metaharness_threat_model metaharness_oia_audit metaharness_audit_list metaharness_audit_trend metaharness_similarity metaharness_drift_from_history metaharness_bench metaharness_evolve metaharness_security_bench metaharness_redblue metaharness_learn metaharness_gepa; do
|
|
2038
2091
|
grep -q "${tool}" "$F" || miss="$miss missing-${tool}"
|
|
2039
2092
|
done
|
|
2040
|
-
#
|
|
2041
|
-
|
|
2093
|
+
# test-mcp-tools.mjs keeps its own hardcoded `tools.length === N` literal —
|
|
2094
|
+
# it is the RUNTIME side of the cross-check. This grep derives N from the
|
|
2095
|
+
# wrapper source, so adding a tool without bumping the test literal fails
|
|
2096
|
+
# here (independent surfaces compared, per the derived-counts contract).
|
|
2097
|
+
grep -q "tools.length === $EXPECTED_TOOLS" "$F" || miss="$miss tool-count-assertion-stale:expected-tools.length===$EXPECTED_TOOLS"
|
|
2042
2098
|
# Graceful skip when dist absent (so the script is smoke-runnable pre-build)
|
|
2043
2099
|
grep -q "SKIPPED" "$F" || miss="$miss no-skip-doc"
|
|
2044
2100
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
@@ -31,11 +31,21 @@
|
|
|
31
31
|
// 2 setup error
|
|
32
32
|
|
|
33
33
|
import { spawnSync } from 'node:child_process';
|
|
34
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
35
|
+
import { tmpdir } from 'node:os';
|
|
34
36
|
import { dirname, join } from 'node:path';
|
|
35
37
|
import { fileURLToPath } from 'node:url';
|
|
36
38
|
|
|
37
39
|
const SCRIPTS_DIR = dirname(fileURLToPath(import.meta.url));
|
|
38
40
|
|
|
41
|
+
// Since the versioned-cache consolidation (_invoke.mjs), a warm
|
|
42
|
+
// ~/.ruflo/<pkg>-cache-<pin> or a locally installed metaharness would satisfy
|
|
43
|
+
// resolution WITHOUT touching the (unreachable) registry — making this drill
|
|
44
|
+
// vacuous on developer machines. Point the cache base at an empty temp dir
|
|
45
|
+
// and disable local walk-up resolution so the install path (and therefore
|
|
46
|
+
// the degraded contract) is actually exercised.
|
|
47
|
+
const EMPTY_CACHE_BASE = mkdtempSync(join(tmpdir(), 'ruflo-degradation-drill-'));
|
|
48
|
+
|
|
39
49
|
const ARGS = (() => {
|
|
40
50
|
const a = { keep: false };
|
|
41
51
|
for (let i = 2; i < process.argv.length; i++) {
|
|
@@ -61,6 +71,8 @@ function runWithUnreachableRegistry(scriptName, extraArgs) {
|
|
|
61
71
|
...process.env,
|
|
62
72
|
npm_config_registry: UNREACHABLE_REGISTRY,
|
|
63
73
|
NPM_CONFIG_REGISTRY: UNREACHABLE_REGISTRY,
|
|
74
|
+
RUFLO_METAHARNESS_CACHE_BASE: EMPTY_CACHE_BASE,
|
|
75
|
+
RUFLO_METAHARNESS_SKIP_LOCAL: '1',
|
|
64
76
|
},
|
|
65
77
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
66
78
|
encoding: 'utf-8',
|
|
@@ -137,6 +149,10 @@ function main() {
|
|
|
137
149
|
assert(/"degraded"\s*:\s*true/.test(r.stdout), `${s.name} emits "degraded": true`);
|
|
138
150
|
}
|
|
139
151
|
|
|
152
|
+
if (!ARGS.keep) {
|
|
153
|
+
try { rmSync(EMPTY_CACHE_BASE, { recursive: true, force: true }); } catch { /* best-effort */ }
|
|
154
|
+
}
|
|
155
|
+
|
|
140
156
|
console.log(`\n${passed} passed, ${failed} failed`);
|
|
141
157
|
if (failed > 0) {
|
|
142
158
|
console.log('\nFailures:');
|
|
@@ -74,7 +74,7 @@ async function main() {
|
|
|
74
74
|
// ──────────────────────────────────────────────────────────────────
|
|
75
75
|
console.log('Phase 1 — module shape');
|
|
76
76
|
assert(Array.isArray(tools), 'metaharnessTools is an array');
|
|
77
|
-
assert(tools.length ===
|
|
77
|
+
assert(tools.length === 15, `15 tools registered (got ${tools.length})`);
|
|
78
78
|
|
|
79
79
|
const expectedNames = new Set([
|
|
80
80
|
'metaharness_score',
|
|
@@ -92,8 +92,12 @@ async function main() {
|
|
|
92
92
|
'metaharness_bench',
|
|
93
93
|
'metaharness_evolve',
|
|
94
94
|
'metaharness_security_bench',
|
|
95
|
-
// @metaharness/redblue@~0.1.
|
|
95
|
+
// @metaharness/redblue@~0.1.4 — adversarial red/blue LLM testing
|
|
96
96
|
'metaharness_redblue',
|
|
97
|
+
// metaharness@0.3.0 — upstream ADR-235 GEPA learning run
|
|
98
|
+
'metaharness_learn',
|
|
99
|
+
// @metaharness/darwin@0.8.0 — GEPA library surface (genome ops)
|
|
100
|
+
'metaharness_gepa',
|
|
97
101
|
]);
|
|
98
102
|
const actualNames = new Set(tools.map((t) => t.name));
|
|
99
103
|
for (const name of expectedNames) {
|
|
@@ -153,6 +157,16 @@ async function main() {
|
|
|
153
157
|
// calls. Count=1 keeps cold-cache npx fetch the dominant cost.
|
|
154
158
|
input = { subcommand: 'attack', family: 'prompt', count: 1 };
|
|
155
159
|
}
|
|
160
|
+
if (tool.name === 'metaharness_learn') {
|
|
161
|
+
// No repo checkout in CI → structured {status:"checkout-required"}
|
|
162
|
+
// exit-0 path. $0: without run=true upstream never spends anyway.
|
|
163
|
+
input = {};
|
|
164
|
+
}
|
|
165
|
+
if (tool.name === 'metaharness_gepa') {
|
|
166
|
+
// op is required; `genome` loads + validates the SHIPPED cand-6
|
|
167
|
+
// genome — pure-local library call once darwin is cached.
|
|
168
|
+
input = { op: 'genome' };
|
|
169
|
+
}
|
|
156
170
|
|
|
157
171
|
// iter 124 → 130 — timeouts have crept up as CI cold-cache npx
|
|
158
172
|
// warmup costs got measured. Final budgets:
|
|
@@ -167,9 +181,14 @@ async function main() {
|
|
|
167
181
|
|| tool.name === 'metaharness_oia_audit'
|
|
168
182
|
|| tool.name === 'metaharness_audit_list'
|
|
169
183
|
// redblue: `attack prompt --count 1` is preview-only (no model
|
|
170
|
-
// calls) but the cold-cache `npx -y @metaharness/redblue@~0.1.
|
|
184
|
+
// calls) but the cold-cache `npx -y @metaharness/redblue@~0.1.4`
|
|
171
185
|
// fetch can take 30-60s. 180s gives 3x headroom.
|
|
172
|
-
|| tool.name === 'metaharness_redblue'
|
|
186
|
+
|| tool.name === 'metaharness_redblue'
|
|
187
|
+
// learn: cold-cache `npx -y metaharness@latest` fetch dominates.
|
|
188
|
+
// gepa: one-time `npm install --prefix ~/.ruflo/darwin-cache-*`
|
|
189
|
+
// fallback install can take 30-60s on cold cache.
|
|
190
|
+
|| tool.name === 'metaharness_learn'
|
|
191
|
+
|| tool.name === 'metaharness_gepa';
|
|
173
192
|
const timeoutMs = isChainTool ? 180_000 : 60_000;
|
|
174
193
|
const handlerPromise = tool.handler(input);
|
|
175
194
|
const timeoutPromise = new Promise((_, reject) =>
|
|
@@ -444,7 +463,7 @@ async function main() {
|
|
|
444
463
|
for (const f of failures) console.log(` - ${f}`);
|
|
445
464
|
process.exit(1);
|
|
446
465
|
}
|
|
447
|
-
console.log('\n✓ All
|
|
466
|
+
console.log('\n✓ All 15 MCP tools satisfy the runtime contract.');
|
|
448
467
|
}
|
|
449
468
|
|
|
450
469
|
main().catch((e) => {
|