@claude-flow/cli 3.16.3 → 3.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/commands/metaharness.d.ts.map +1 -1
- package/dist/src/commands/metaharness.js +9 -3
- package/dist/src/commands/metaharness.js.map +1 -1
- package/dist/src/mcp-tools/metaharness-tools.d.ts +7 -0
- package/dist/src/mcp-tools/metaharness-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/metaharness-tools.js +89 -0
- package/dist/src/mcp-tools/metaharness-tools.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +5 -5
- package/plugins/ruflo-metaharness/README.md +2 -0
- package/plugins/ruflo-metaharness/scripts/_darwin.mjs +5 -3
- package/plugins/ruflo-metaharness/scripts/_redblue.mjs +10 -4
- package/plugins/ruflo-metaharness/scripts/gepa.mjs +199 -0
- package/plugins/ruflo-metaharness/scripts/learn.mjs +127 -0
- package/plugins/ruflo-metaharness/scripts/smoke.sh +26 -21
- package/plugins/ruflo-metaharness/scripts/test-mcp-tools.mjs +24 -5
- package/plugins/ruflo-metaharness/skills/harness-evolve/SKILL.md +1 -1
- package/plugins/ruflo-metaharness/skills/harness-gepa/SKILL.md +65 -0
- package/plugins/ruflo-metaharness/skills/harness-learn/SKILL.md +63 -0
- package/plugins/ruflo-metaharness/skills/harness-security-bench/SKILL.md +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@claude-flow/cli",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.17.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Ruflo CLI - Enterprise AI agent orchestration with 60+ specialized agents, swarm coordination, MCP server, self-learning hooks, and vector memory for Claude Code",
|
|
6
6
|
"main": "dist/src/index.js",
|
|
@@ -115,11 +115,11 @@
|
|
|
115
115
|
"@claude-flow/memory": "^3.0.0-alpha.21",
|
|
116
116
|
"@claude-flow/plugin-gastown-bridge": "^0.1.3",
|
|
117
117
|
"@claude-flow/security": "^3.0.0-alpha.10",
|
|
118
|
-
"@metaharness/darwin": "~0.
|
|
119
|
-
"@metaharness/kernel": "~0.1.
|
|
120
|
-
"@metaharness/redblue": "~0.1.
|
|
118
|
+
"@metaharness/darwin": "~0.8.0",
|
|
119
|
+
"@metaharness/kernel": "~0.1.2",
|
|
120
|
+
"@metaharness/redblue": "~0.1.4",
|
|
121
121
|
"@metaharness/router": "~0.3.2",
|
|
122
|
-
"metaharness": "~0.2.
|
|
122
|
+
"metaharness": "~0.2.8",
|
|
123
123
|
"@ruvector/attention": "^0.1.32",
|
|
124
124
|
"@ruvector/attention-darwin-arm64": "0.1.32",
|
|
125
125
|
"@ruvector/diskann": "^0.1.0",
|
|
@@ -26,6 +26,8 @@ MetaHarness integration plugin for ruflo. Surfaces the upstream `metaharness` /
|
|
|
26
26
|
| `harness-bench` | `/harness-bench --op create\|verify --repo <path>` | Manage `@metaharness/darwin` bench suites — fixed evaluation corpora for `harness-evolve` |
|
|
27
27
|
| `harness-evolve` | `/harness-evolve --repo <path> [--generations 3] [--sandbox real\|mock\|agent]` | Run `@metaharness/darwin evolve` — mutate seven policy surfaces, sandbox-score variants, promote measured wins |
|
|
28
28
|
| `harness-security-bench` | `/harness-security-bench [--population 2] [--cycles 1] [--alert-on-fail]` | "Darwin Shield" / ADR-155 — evolve a security-detection harness against a 10-vuln corpus |
|
|
29
|
+
| `harness-learn` | `/harness-learn --host <h> --model <m> --slice <manifest> [--repo <checkout>] [--run]` | metaharness@0.3.0 / upstream ADR-235 — GEPA learning run; $0 dry-run default, `--run` to spend; needs a metaharness repo checkout |
|
|
30
|
+
| `harness-gepa` | `/harness-gepa --op genome\|validate\|render\|analyze [--path <genome.json>]` | darwin@0.8.0 GEPA library surface — genome load/validate/render + transcript failure analysis; `gepaOptimize` stays library-only |
|
|
29
31
|
|
|
30
32
|
## Phase-0 baseline (ruflo itself, 2026-06-16)
|
|
31
33
|
|
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
//
|
|
3
3
|
// Mirrors `_harness.mjs` for the umbrella `metaharness` / `harness` binaries,
|
|
4
4
|
// but targets the separate darwin binary (`metaharness-darwin`) which is
|
|
5
|
-
// published as its own npm package (`@metaharness/darwin@~0.
|
|
5
|
+
// published as its own npm package (`@metaharness/darwin@~0.8.0`).
|
|
6
6
|
//
|
|
7
|
-
// Three subcommands surfaced (matches darwin 0.3.x
|
|
7
|
+
// Three subcommands surfaced (matches darwin 0.8.x — same verbs as 0.3.x,
|
|
8
|
+
// with GEPA-engine evolve flags added upstream: --selection modes,
|
|
9
|
+
// --crossover, --epistasis, --curriculum, --mutator ruvllm, --sandbox):
|
|
8
10
|
// - `metaharness-darwin evolve <repo> [...]` — harness self-improvement
|
|
9
11
|
// - `metaharness-darwin bench <create|verify> ...` — bench-suite lifecycle
|
|
10
12
|
// - `metaharness-darwin security bench [...]` — Darwin Shield (upstream ADR-155)
|
|
@@ -32,7 +34,7 @@ const DEFAULT_TIMEOUT_MS = 60_000;
|
|
|
32
34
|
// Pinned semver range. Bump in lock-step with optionalDependencies in
|
|
33
35
|
// @claude-flow/cli/package.json + ruflo/package.json. The `~` allows
|
|
34
36
|
// patch upgrades without re-pinning.
|
|
35
|
-
const DARWIN_PIN = '@metaharness/darwin@~0.
|
|
37
|
+
const DARWIN_PIN = '@metaharness/darwin@~0.8.0';
|
|
36
38
|
|
|
37
39
|
const DEGRADED_RX = /could not determine executable|404|not installed|MODULE_NOT_FOUND|ENOTFOUND|getaddrinfo|ECONNREFUSED|ETIMEDOUT/i;
|
|
38
40
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// _redblue.mjs — invocation helper for `@metaharness/redblue`.
|
|
2
2
|
//
|
|
3
3
|
// Sibling of `_darwin.mjs` / `_harness.mjs`. Targets the `redblue` binary
|
|
4
|
-
// from the standalone `@metaharness/redblue@~0.1.
|
|
4
|
+
// from the standalone `@metaharness/redblue@~0.1.4` package.
|
|
5
5
|
//
|
|
6
6
|
// Subcommands surfaced (matches redblue 0.1.x):
|
|
7
7
|
// - `redblue init [--out redblue.yaml]`
|
|
@@ -52,11 +52,17 @@ const INSTALL_TIMEOUT_MS = 180_000; // npm install can be slow on cold cache
|
|
|
52
52
|
|
|
53
53
|
// Pinned semver range. Bump in lock-step with optionalDependencies in
|
|
54
54
|
// @claude-flow/cli/package.json + ruflo/package.json.
|
|
55
|
-
const REDBLUE_PIN = '@metaharness/redblue@~0.1.
|
|
55
|
+
const REDBLUE_PIN = '@metaharness/redblue@~0.1.4';
|
|
56
56
|
const REDBLUE_PKG = '@metaharness/redblue';
|
|
57
|
-
const REDBLUE_PIN_VERSION = '~0.1.
|
|
57
|
+
const REDBLUE_PIN_VERSION = '~0.1.4';
|
|
58
58
|
|
|
59
|
-
|
|
59
|
+
// Cache dir is versioned by the pin so bumping REDBLUE_PIN_VERSION
|
|
60
|
+
// invalidates stale installs — ensureInstalled() short-circuits on
|
|
61
|
+
// existsSync(RESOLVED_CLI), so an unversioned dir would keep serving
|
|
62
|
+
// the old package forever after a pin bump.
|
|
63
|
+
const CACHE_DIR = join(
|
|
64
|
+
homedir(), '.ruflo', `redblue-cache-${REDBLUE_PIN_VERSION.replace(/[~^]/g, '')}`,
|
|
65
|
+
);
|
|
60
66
|
const RESOLVED_CLI = join(
|
|
61
67
|
CACHE_DIR, 'node_modules', '@metaharness', 'redblue', 'dist', 'cli', 'index.js',
|
|
62
68
|
);
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// gepa.mjs — surfaces the `@metaharness/darwin/gepa` LIBRARY exports.
|
|
3
|
+
//
|
|
4
|
+
// Unlike every other script in this plugin, gepa has no CLI equivalent —
|
|
5
|
+
// GEPA (darwin 0.8.0's genetic-evolution prompt-adaptation engine) ships as
|
|
6
|
+
// a library entry (`import { ... } from '@metaharness/darwin/gepa'`). This
|
|
7
|
+
// script wraps the subprocess-safe subset:
|
|
8
|
+
//
|
|
9
|
+
// genome load + validate a genome (default: the shipped cand-6 — the
|
|
10
|
+
// first holdout-confirmed cheap-tier policy promotion)
|
|
11
|
+
// validate validateGenome(json) → structural errors[]
|
|
12
|
+
// render buildSystemFromGenome(genome) → the system prompt a genome
|
|
13
|
+
// compiles to (inspect what a policy actually says)
|
|
14
|
+
// analyze analyzeTranscript(entries) → failure-class breakdown
|
|
15
|
+
//
|
|
16
|
+
// NOT SURFACED: `gepaOptimize` — it takes an in-process `evaluate(candidate)`
|
|
17
|
+
// callback ("bring your own evaluator") which cannot cross a subprocess
|
|
18
|
+
// boundary. Optimization runs belong either in library consumers
|
|
19
|
+
// (import '@metaharness/darwin/gepa' directly) or behind the darwin CLI's
|
|
20
|
+
// `evolve` verb (scripts/evolve.mjs), which pairs GEPA with its sandbox
|
|
21
|
+
// evaluators.
|
|
22
|
+
//
|
|
23
|
+
// MODULE RESOLUTION (ADR-150 graceful degradation)
|
|
24
|
+
// ================================================
|
|
25
|
+
// 1. Try bare `import('@metaharness/darwin/gepa')` — free when the optional
|
|
26
|
+
// dep is installed in an ancestor node_modules.
|
|
27
|
+
// 2. Fall back to a ruflo-owned versioned cache install
|
|
28
|
+
// (~/.ruflo/darwin-cache-<pin>) — same pattern as _redblue.mjs; the
|
|
29
|
+
// versioned dir means pin bumps invalidate stale caches automatically.
|
|
30
|
+
// 3. Both fail → `{degraded: true}` exit 0. Never throws.
|
|
31
|
+
//
|
|
32
|
+
// EXIT CODES
|
|
33
|
+
// 0 op completed (or degraded)
|
|
34
|
+
// 1 --alert-on-invalid and validate found errors
|
|
35
|
+
// 2 config error (bad op / missing file)
|
|
36
|
+
|
|
37
|
+
import { readFileSync, existsSync, mkdirSync } from 'node:fs';
|
|
38
|
+
import { spawnSync } from 'node:child_process';
|
|
39
|
+
import { homedir } from 'node:os';
|
|
40
|
+
import { join } from 'node:path';
|
|
41
|
+
import { pathToFileURL } from 'node:url';
|
|
42
|
+
|
|
43
|
+
// Bump in lock-step with DARWIN_PIN in _darwin.mjs + optionalDependencies.
|
|
44
|
+
const DARWIN_PIN_VERSION = '~0.8.0';
|
|
45
|
+
const CACHE_DIR = join(
|
|
46
|
+
homedir(), '.ruflo', `darwin-cache-${DARWIN_PIN_VERSION.replace(/[~^]/g, '')}`,
|
|
47
|
+
);
|
|
48
|
+
const CACHED_GEPA = join(
|
|
49
|
+
CACHE_DIR, 'node_modules', '@metaharness', 'darwin', 'dist', 'gepa', 'index.js',
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
const ARGS = (() => {
|
|
53
|
+
const a = {
|
|
54
|
+
op: null,
|
|
55
|
+
path: null, // genome JSON path (genome/validate/render); default cand-6
|
|
56
|
+
transcript: null, // transcript JSON path (analyze)
|
|
57
|
+
ext: undefined, // render — target file extension hint
|
|
58
|
+
glob: undefined, // render — target glob hint
|
|
59
|
+
alertOnInvalid: false,
|
|
60
|
+
format: 'json',
|
|
61
|
+
};
|
|
62
|
+
for (let i = 2; i < process.argv.length; i++) {
|
|
63
|
+
const v = process.argv[i];
|
|
64
|
+
if (v === '--op') a.op = process.argv[++i];
|
|
65
|
+
else if (v === '--path') a.path = process.argv[++i];
|
|
66
|
+
else if (v === '--transcript') a.transcript = process.argv[++i];
|
|
67
|
+
else if (v === '--ext') a.ext = process.argv[++i];
|
|
68
|
+
else if (v === '--glob') a.glob = process.argv[++i];
|
|
69
|
+
else if (v === '--alert-on-invalid') a.alertOnInvalid = true;
|
|
70
|
+
else if (v === '--format') a.format = process.argv[++i];
|
|
71
|
+
}
|
|
72
|
+
return a;
|
|
73
|
+
})();
|
|
74
|
+
|
|
75
|
+
function emitDegradedAndExit(reason) {
|
|
76
|
+
console.log(JSON.stringify({
|
|
77
|
+
degraded: true,
|
|
78
|
+
reason,
|
|
79
|
+
hint: 'Install with `npm i -D @metaharness/darwin@' + DARWIN_PIN_VERSION
|
|
80
|
+
+ '` or verify network access — the gepa entry ships inside the darwin package.',
|
|
81
|
+
generatedAt: new Date().toISOString(),
|
|
82
|
+
}, null, 2));
|
|
83
|
+
process.exit(0); // ADR-150 — ruflo stays operational without MetaHarness
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function importGepa() {
|
|
87
|
+
try {
|
|
88
|
+
return await import('@metaharness/darwin/gepa');
|
|
89
|
+
} catch (e) {
|
|
90
|
+
const msg = String(e?.message ?? e);
|
|
91
|
+
// Fall back on absence AND on stale installs: a pre-0.8.0 darwin in an
|
|
92
|
+
// ancestor node_modules has no './gepa' subpath, which throws
|
|
93
|
+
// ERR_PACKAGE_PATH_NOT_EXPORTED rather than MODULE_NOT_FOUND.
|
|
94
|
+
const recoverable = /Cannot find (module|package)|ERR_MODULE_NOT_FOUND|MODULE_NOT_FOUND|ERR_PACKAGE_PATH_NOT_EXPORTED|is not defined by "exports"/i;
|
|
95
|
+
if (!recoverable.test(msg)) throw e;
|
|
96
|
+
}
|
|
97
|
+
// Cached-install fallback.
|
|
98
|
+
if (!existsSync(CACHED_GEPA)) {
|
|
99
|
+
try {
|
|
100
|
+
mkdirSync(CACHE_DIR, { recursive: true });
|
|
101
|
+
} catch {
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
const r = spawnSync('npm', [
|
|
105
|
+
'install',
|
|
106
|
+
'--no-audit', '--no-fund', '--no-package-lock',
|
|
107
|
+
'--prefix', CACHE_DIR,
|
|
108
|
+
`@metaharness/darwin@${DARWIN_PIN_VERSION}`,
|
|
109
|
+
], {
|
|
110
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
111
|
+
encoding: 'utf-8',
|
|
112
|
+
timeout: 180_000,
|
|
113
|
+
shell: process.platform === 'win32',
|
|
114
|
+
});
|
|
115
|
+
if (r.status !== 0 || !existsSync(CACHED_GEPA)) return null;
|
|
116
|
+
}
|
|
117
|
+
try {
|
|
118
|
+
return await import(pathToFileURL(CACHED_GEPA).href);
|
|
119
|
+
} catch {
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function readJsonFile(path, label) {
|
|
125
|
+
if (!path || !existsSync(path)) {
|
|
126
|
+
console.error(`gepa: ${label} file not found: ${path}`);
|
|
127
|
+
process.exit(2);
|
|
128
|
+
}
|
|
129
|
+
try {
|
|
130
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
131
|
+
} catch (e) {
|
|
132
|
+
console.error(`gepa: ${label} is not valid JSON: ${e?.message ?? e}`);
|
|
133
|
+
process.exit(2);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function loadGenomeOrExit(gepa) {
|
|
138
|
+
if (ARGS.path) {
|
|
139
|
+
if (!existsSync(ARGS.path)) {
|
|
140
|
+
console.error(`gepa: --path genome file not found: ${ARGS.path}`);
|
|
141
|
+
process.exit(2);
|
|
142
|
+
}
|
|
143
|
+
// upstream signature: loadGenome(readFileSync, path) — fs injected.
|
|
144
|
+
return { genome: gepa.loadGenome(readFileSync, ARGS.path), source: ARGS.path };
|
|
145
|
+
}
|
|
146
|
+
return { genome: gepa.loadCand6Genome(), source: gepa.CAND6_GENOME_PATH };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async function main() {
|
|
150
|
+
const OPS = ['genome', 'validate', 'render', 'analyze'];
|
|
151
|
+
if (!OPS.includes(ARGS.op)) {
|
|
152
|
+
console.error(`gepa: --op must be one of ${OPS.join('|')}`);
|
|
153
|
+
process.exit(2);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const gepa = await importGepa();
|
|
157
|
+
if (!gepa) emitDegradedAndExit('metaharness-darwin-not-available');
|
|
158
|
+
|
|
159
|
+
const start = Date.now();
|
|
160
|
+
let out;
|
|
161
|
+
|
|
162
|
+
if (ARGS.op === 'genome') {
|
|
163
|
+
const { genome, source } = loadGenomeOrExit(gepa);
|
|
164
|
+
const errors = gepa.validateGenome(genome);
|
|
165
|
+
out = { op: 'genome', source, valid: errors.length === 0, errors, genome };
|
|
166
|
+
} else if (ARGS.op === 'validate') {
|
|
167
|
+
// validate takes raw JSON (not loadGenome) so structurally-broken files
|
|
168
|
+
// reach validateGenome instead of throwing in the loader.
|
|
169
|
+
const raw = ARGS.path
|
|
170
|
+
? readJsonFile(ARGS.path, '--path genome')
|
|
171
|
+
: gepa.loadCand6Genome();
|
|
172
|
+
const errors = gepa.validateGenome(raw);
|
|
173
|
+
out = { op: 'validate', source: ARGS.path ?? gepa.CAND6_GENOME_PATH, valid: errors.length === 0, errors };
|
|
174
|
+
} else if (ARGS.op === 'render') {
|
|
175
|
+
const { genome, source } = loadGenomeOrExit(gepa);
|
|
176
|
+
const system = gepa.buildSystemFromGenome(genome, ARGS.ext, ARGS.glob);
|
|
177
|
+
out = { op: 'render', source, chars: system.length, system };
|
|
178
|
+
} else {
|
|
179
|
+
// analyze
|
|
180
|
+
const entries = readJsonFile(ARGS.transcript, '--transcript');
|
|
181
|
+
if (!Array.isArray(entries)) {
|
|
182
|
+
console.error('gepa: --transcript must be a JSON array of transcript entries');
|
|
183
|
+
process.exit(2);
|
|
184
|
+
}
|
|
185
|
+
const analysis = gepa.analyzeTranscript(entries);
|
|
186
|
+
out = { op: 'analyze', source: ARGS.transcript, entries: entries.length, analysis };
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
out.durationMs = Date.now() - start;
|
|
190
|
+
console.log(JSON.stringify(out, null, 2));
|
|
191
|
+
|
|
192
|
+
if (ARGS.alertOnInvalid && out.valid === false) process.exit(1);
|
|
193
|
+
process.exit(0);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
main().catch((e) => {
|
|
197
|
+
console.error(`gepa: ${e?.message ?? e}`);
|
|
198
|
+
process.exit(2);
|
|
199
|
+
});
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// learn.mjs — wrapper around `metaharness learn` (upstream ADR-235, metaharness@0.3.0).
|
|
3
|
+
//
|
|
4
|
+
// GEPA learning run: optimizes a harness genome against a SWE-bench-style
|
|
5
|
+
// slice manifest. $0 DRY-RUN BY DEFAULT — upstream only spends (model calls,
|
|
6
|
+
// Docker sandboxes) when --run is passed, and we forward that flag verbatim
|
|
7
|
+
// so the spend opt-in stays explicit at every layer.
|
|
8
|
+
//
|
|
9
|
+
// CHECKOUT PRECONDITION (upstream design, not a bug)
|
|
10
|
+
// ==================================================
|
|
11
|
+
// The learning harness (GEPA + SWE-bench + Docker) is too heavy to ship in
|
|
12
|
+
// the npm package, so `metaharness learn` requires a local clone of the
|
|
13
|
+
// metaharness repo, located via $METAHARNESS_REPO or by running inside the
|
|
14
|
+
// clone. When the checkout is absent we emit a structured
|
|
15
|
+
// `{status: "checkout-required"}` payload and exit 0 — the script ran as
|
|
16
|
+
// designed and told the agent what to do next. This is distinct from
|
|
17
|
+
// `degraded: true` (npm package absent). The managed-service path (gateway-
|
|
18
|
+
// side learn jobs, no checkout) is upstream's ADR-235 follow-up.
|
|
19
|
+
//
|
|
20
|
+
// USAGE
|
|
21
|
+
// node scripts/learn.mjs --host claude-code --model haiku --slice slices/lite.json
|
|
22
|
+
// node scripts/learn.mjs --repo ~/src/metaharness --host codex --model gpt-5-mini --slice s.json --run
|
|
23
|
+
//
|
|
24
|
+
// EXIT CODES
|
|
25
|
+
// 0 learn completed (or dry-run report, or degraded, or checkout-required)
|
|
26
|
+
// 1 --alert-on-fail and the learn run reported failure
|
|
27
|
+
// 2 config error (bad arg)
|
|
28
|
+
|
|
29
|
+
import { existsSync } from 'node:fs';
|
|
30
|
+
import { runMetaharnessAsync, emitDegradedJsonAndExit } from './_harness.mjs';
|
|
31
|
+
|
|
32
|
+
const ARGS = (() => {
|
|
33
|
+
const a = {
|
|
34
|
+
host: null,
|
|
35
|
+
model: null,
|
|
36
|
+
slice: null,
|
|
37
|
+
repo: null,
|
|
38
|
+
run: false,
|
|
39
|
+
alertOnFail: false,
|
|
40
|
+
format: 'json',
|
|
41
|
+
timeoutMs: null,
|
|
42
|
+
};
|
|
43
|
+
for (let i = 2; i < process.argv.length; i++) {
|
|
44
|
+
const v = process.argv[i];
|
|
45
|
+
if (v === '--host') a.host = process.argv[++i];
|
|
46
|
+
else if (v === '--model') a.model = process.argv[++i];
|
|
47
|
+
else if (v === '--slice') a.slice = process.argv[++i];
|
|
48
|
+
else if (v === '--repo') a.repo = process.argv[++i];
|
|
49
|
+
else if (v === '--run') a.run = true;
|
|
50
|
+
else if (v === '--alert-on-fail') a.alertOnFail = true;
|
|
51
|
+
else if (v === '--format') a.format = process.argv[++i];
|
|
52
|
+
else if (v === '--timeout-ms') a.timeoutMs = parseInt(process.argv[++i], 10);
|
|
53
|
+
}
|
|
54
|
+
return a;
|
|
55
|
+
})();
|
|
56
|
+
|
|
57
|
+
const CHECKOUT_RX = /requires a metaharness repo checkout/i;
|
|
58
|
+
|
|
59
|
+
function defaultTimeoutMs() {
|
|
60
|
+
// Dry-run resolves the slice manifest + prices the run without spending —
|
|
61
|
+
// bounded by repo scan, not model calls. Real runs (--run) are dominated
|
|
62
|
+
// by model calls × slice size; callers should pass --timeout-ms matched
|
|
63
|
+
// to their slice. 10 min is a floor for small slices, not a budget.
|
|
64
|
+
return ARGS.run ? 600_000 : 120_000;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function main() {
|
|
68
|
+
if (ARGS.repo && !existsSync(ARGS.repo)) {
|
|
69
|
+
console.error(`learn: --repo path does not exist: ${ARGS.repo}`);
|
|
70
|
+
process.exit(2);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const cliArgs = ['learn'];
|
|
74
|
+
if (ARGS.host) cliArgs.push('--host', ARGS.host);
|
|
75
|
+
if (ARGS.model) cliArgs.push('--model', ARGS.model);
|
|
76
|
+
if (ARGS.slice) cliArgs.push('--slice', ARGS.slice);
|
|
77
|
+
if (ARGS.run) cliArgs.push('--run');
|
|
78
|
+
|
|
79
|
+
const r = await runMetaharnessAsync(cliArgs, {
|
|
80
|
+
// learn emits a human-readable report; --json support is not guaranteed
|
|
81
|
+
// across 0.3.x, so we don't inject it and parse leniently instead.
|
|
82
|
+
json: false,
|
|
83
|
+
timeoutMs: ARGS.timeoutMs ?? defaultTimeoutMs(),
|
|
84
|
+
env: ARGS.repo ? { METAHARNESS_REPO: ARGS.repo } : {},
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
if (r.degraded) emitDegradedJsonAndExit(r.reason);
|
|
88
|
+
|
|
89
|
+
const combined = `${r.stdout}\n${r.stderr}`;
|
|
90
|
+
if (CHECKOUT_RX.test(combined)) {
|
|
91
|
+
console.log(JSON.stringify({
|
|
92
|
+
status: 'checkout-required',
|
|
93
|
+
hint: 'git clone https://github.com/ruvnet/metaharness.git, then re-run with '
|
|
94
|
+
+ '--repo /path/to/metaharness (or set METAHARNESS_REPO). The learning '
|
|
95
|
+
+ 'harness (GEPA + SWE-bench + Docker) is not shipped in the npm package.',
|
|
96
|
+
dryRun: !ARGS.run,
|
|
97
|
+
durationMs: r.durationMs,
|
|
98
|
+
}, null, 2));
|
|
99
|
+
process.exit(0);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Lenient JSON extraction — grab the last {...} block if one exists.
|
|
103
|
+
let json = null;
|
|
104
|
+
const matches = [...r.stdout.matchAll(/\{[\s\S]*?\}/g)];
|
|
105
|
+
for (let i = matches.length - 1; i >= 0; i--) {
|
|
106
|
+
try { json = JSON.parse(matches[i][0]); break; } catch { /* try previous */ }
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const out = {
|
|
110
|
+
status: r.exitCode === 0 ? 'ok' : 'failed',
|
|
111
|
+
dryRun: !ARGS.run,
|
|
112
|
+
exitCode: r.exitCode,
|
|
113
|
+
report: json,
|
|
114
|
+
// When upstream emits no JSON, the raw report is still the deliverable.
|
|
115
|
+
rawReport: json ? undefined : r.stdout.slice(0, 20_000),
|
|
116
|
+
durationMs: r.durationMs,
|
|
117
|
+
};
|
|
118
|
+
console.log(JSON.stringify(out, null, 2));
|
|
119
|
+
|
|
120
|
+
if (ARGS.alertOnFail && r.exitCode !== 0) process.exit(1);
|
|
121
|
+
process.exit(0);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
main().catch((e) => {
|
|
125
|
+
console.error(`learn: ${e?.message ?? e}`);
|
|
126
|
+
process.exit(2);
|
|
127
|
+
});
|
|
@@ -151,9 +151,9 @@ F="$ROOT/../../v3/@claude-flow/cli/src/commands/metaharness.ts"
|
|
|
151
151
|
miss=""
|
|
152
152
|
[[ -f "$F" ]] || miss="$miss command-file-missing"
|
|
153
153
|
grep -q "name: 'metaharness'" "$F" 2>/dev/null || miss="$miss no-name-field"
|
|
154
|
-
# All
|
|
154
|
+
# All core subcommands must each be present in the dispatch table.
|
|
155
155
|
# Match either quoted ('mcp-scan': ...) or unquoted shorthand (score: ...) keys.
|
|
156
|
-
for sub in score genome mcp-scan threat-model oia-audit audit-list audit-trend mint redblue; do
|
|
156
|
+
for sub in score genome mcp-scan threat-model oia-audit audit-list audit-trend mint redblue learn gepa; do
|
|
157
157
|
grep -qE "(^|[[:space:]])'?${sub}'?:" "$F" 2>/dev/null || miss="$miss missing-$sub"
|
|
158
158
|
done
|
|
159
159
|
# Registered in the loader
|
|
@@ -722,7 +722,7 @@ for k in $KEYS; do
|
|
|
722
722
|
grep -qE "npx ruflo metaharness ${k}([ \\\\]|$)" "$CMD" 2>/dev/null \
|
|
723
723
|
|| miss="$miss subcommand-${k}-not-in-claude-md"
|
|
724
724
|
done
|
|
725
|
-
[[ "$COUNT" == "
|
|
725
|
+
[[ "$COUNT" == "13" ]] || miss="$miss subcommand-count-stale:$COUNT-expected-13"
|
|
726
726
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
727
727
|
|
|
728
728
|
step "17z56. every MCP tool documented in CLAUDE.md (iter 93)"
|
|
@@ -742,11 +742,12 @@ for t in $TOOLS; do
|
|
|
742
742
|
grep -q "mcp__claude-flow__${t}" "$CMD" 2>/dev/null \
|
|
743
743
|
|| miss="$miss ${t}-not-in-claude-md"
|
|
744
744
|
done
|
|
745
|
-
# Lock count:
|
|
745
|
+
# Lock count: 15 MCP tools (mint deliberately excluded — see iter 73).
|
|
746
746
|
# Bumped from 9 → 12 after ADR-153 added metaharness_bench,
|
|
747
747
|
# metaharness_evolve, and metaharness_security_bench. Bumped from 12 → 13
|
|
748
748
|
# after @metaharness/redblue integration added metaharness_redblue.
|
|
749
|
-
|
|
749
|
+
# Bumped from 13 → 15 after metaharness@0.3.0 learn + darwin@0.8.0 gepa.
|
|
750
|
+
[[ "$COUNT" == "15" ]] || miss="$miss mcp-tool-count-stale:$COUNT-expected-15"
|
|
750
751
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
751
752
|
|
|
752
753
|
step "17z55. MCP enum + SEVERITY_RANK vocabulary aligned (iter 92)"
|
|
@@ -821,10 +822,11 @@ for f in $REFS; do
|
|
|
821
822
|
COUNT=$((COUNT + 1))
|
|
822
823
|
[[ -f "$SCRIPTS_DIR/$f" ]] || miss="$miss mcp-script-${f}-missing"
|
|
823
824
|
done
|
|
824
|
-
# Should be
|
|
825
|
+
# Should be 15 unique scripts (one per MCP tool; mint deliberately excluded).
|
|
825
826
|
# Bumped from 9 → 12 after ADR-153 added bench/evolve/security_bench tools.
|
|
826
827
|
# Bumped from 12 → 13 after @metaharness/redblue integration added redblue.mjs.
|
|
827
|
-
|
|
828
|
+
# Bumped from 13 → 15 after metaharness@0.3.0 learn + darwin@0.8.0 gepa.
|
|
829
|
+
[[ "$COUNT" == "15" ]] || miss="$miss mcp-script-count-stale:$COUNT-expected-15"
|
|
828
830
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
829
831
|
|
|
830
832
|
step "17z52. SUBCOMMANDS map entries point at existing script files (iter 89)"
|
|
@@ -844,9 +846,9 @@ for f in $MAPPINGS; do
|
|
|
844
846
|
COUNT=$((COUNT + 1))
|
|
845
847
|
[[ -f "$SCRIPTS_DIR/$f" ]] || miss="$miss script-${f}-missing"
|
|
846
848
|
done
|
|
847
|
-
# Iter 73's description string + redblue
|
|
848
|
-
# SUBCOMMANDS map should have the same
|
|
849
|
-
[[ "$COUNT" == "
|
|
849
|
+
# Iter 73's description string + redblue + learn/gepa lists 13 subcommands.
|
|
850
|
+
# SUBCOMMANDS map should have the same 13 entries. Lock the count at exactly 13.
|
|
851
|
+
[[ "$COUNT" == "13" ]] || miss="$miss mapping-count-stale:$COUNT-expected-13"
|
|
850
852
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
851
853
|
|
|
852
854
|
step "17z51. all metaharness scripts produce parseable JSON in --format json mode (iter 88)"
|
|
@@ -1140,9 +1142,9 @@ step "17z36. CLI subcommand list current + mint anti-MCP guard (iter 73)"
|
|
|
1140
1142
|
miss=""
|
|
1141
1143
|
DISP="$ROOT/../../v3/@claude-flow/cli/src/commands/metaharness.ts"
|
|
1142
1144
|
WRAPPER="$ROOT/../../v3/@claude-flow/cli/src/mcp-tools/metaharness-tools.ts"
|
|
1143
|
-
# All
|
|
1144
|
-
#
|
|
1145
|
-
for sub in score genome mcp-scan threat-model oia-audit audit-list audit-trend similarity drift-from-history mint redblue; do
|
|
1145
|
+
# All 13 dispatchable subcommands present (iter-73 had 10; @metaharness/redblue
|
|
1146
|
+
# added redblue.mjs; metaharness@0.3.0/darwin@0.8.0 added learn.mjs + gepa.mjs).
|
|
1147
|
+
for sub in score genome mcp-scan threat-model oia-audit audit-list audit-trend similarity drift-from-history mint redblue learn gepa; do
|
|
1146
1148
|
grep -q "${sub}" "$DISP" 2>/dev/null || miss="$miss subcommand-${sub}-not-listed"
|
|
1147
1149
|
done
|
|
1148
1150
|
# ANTI-MINT GUARD: mint is intentionally CLI-only per ADR-150 §Sandboxing
|
|
@@ -1632,8 +1634,9 @@ WRAPPER="$ROOT/../../v3/@claude-flow/cli/src/mcp-tools/metaharness-tools.ts"
|
|
|
1632
1634
|
# Iter 46 set this at 9 (8 tools); iter 54 added the 9th → 10. ADR-153
|
|
1633
1635
|
# added 3 more tools (bench/evolve/security_bench) → 1 + 12 = 13.
|
|
1634
1636
|
# @metaharness/redblue integration added 1 more → 1 + 13 = 14.
|
|
1637
|
+
# learn + gepa (metaharness@0.3.0 / darwin@0.8.0) added 2 more → 1 + 15 = 16.
|
|
1635
1638
|
COUNT=$(grep -c "MCP_SUCCESS_SEMANTIC" "$WRAPPER" 2>/dev/null; true)
|
|
1636
|
-
[[ "$COUNT" == "
|
|
1639
|
+
[[ "$COUNT" == "16" ]] || miss="$miss footnote-count:$COUNT-expected-16"
|
|
1637
1640
|
# audit_trend now exposes baselineFile / currentFile
|
|
1638
1641
|
grep -q "baselineFile" "$WRAPPER" 2>/dev/null || miss="$miss no-baseline-file"
|
|
1639
1642
|
grep -q "currentFile" "$WRAPPER" 2>/dev/null || miss="$miss no-current-file"
|
|
@@ -1673,8 +1676,8 @@ COUNT_OLD=$(grep -c "success: !r.degraded" "$WRAPPER" 2>/dev/null; true)
|
|
|
1673
1676
|
COUNT_NEW=$(grep -c "success: r.success" "$WRAPPER" 2>/dev/null; true)
|
|
1674
1677
|
# Iter 54 added a 9th tool → 9. ADR-153 added 3 more (bench, evolve,
|
|
1675
1678
|
# security_bench) → 12. @metaharness/redblue integration added 1 more → 13.
|
|
1676
|
-
# Future iters that add tools should bump this.
|
|
1677
|
-
[[ "$COUNT_NEW" == "
|
|
1679
|
+
# learn + gepa added 2 more → 15. Future iters that add tools should bump this.
|
|
1680
|
+
[[ "$COUNT_NEW" == "15" ]] || miss="$miss new-pattern-count:$COUNT_NEW-expected-15"
|
|
1678
1681
|
# Runtime anchors: iter 44 success assertions present
|
|
1679
1682
|
T="$ROOT/scripts/test-mcp-tools.mjs"
|
|
1680
1683
|
grep -q "iter 44 fix" "$T" 2>/dev/null || miss="$miss no-iter44-anchors"
|
|
@@ -2032,13 +2035,15 @@ grep -q "result has 'success'" "$F" || miss="$miss no-success-assertion"
|
|
|
2032
2035
|
grep -q "result has 'data'" "$F" || miss="$miss no-data-assertion"
|
|
2033
2036
|
grep -q "result has 'degraded'" "$F" || miss="$miss no-degraded-assertion"
|
|
2034
2037
|
grep -q "result has 'exitCode'" "$F" || miss="$miss no-exitcode-assertion"
|
|
2035
|
-
# All
|
|
2036
|
-
# ADR-153 added bench/evolve/security_bench, @metaharness/redblue added redblue
|
|
2037
|
-
|
|
2038
|
+
# All 15 tool names enumerated (similarity iter 36, drift_from_history iter 54,
|
|
2039
|
+
# ADR-153 added bench/evolve/security_bench, @metaharness/redblue added redblue,
|
|
2040
|
+
# metaharness@0.3.0/darwin@0.8.0 added learn + gepa)
|
|
2041
|
+
for tool in metaharness_score metaharness_genome metaharness_mcp_scan metaharness_threat_model metaharness_oia_audit metaharness_audit_list metaharness_audit_trend metaharness_similarity metaharness_drift_from_history metaharness_bench metaharness_evolve metaharness_security_bench metaharness_redblue metaharness_learn metaharness_gepa; do
|
|
2038
2042
|
grep -q "${tool}" "$F" || miss="$miss missing-${tool}"
|
|
2039
2043
|
done
|
|
2040
|
-
# Count assertion must match iter-54 (8 → 9), then ADR-153 (9 → 12), then
|
|
2041
|
-
|
|
2044
|
+
# Count assertion must match iter-54 (8 → 9), then ADR-153 (9 → 12), then
|
|
2045
|
+
# redblue (12 → 13), then learn/gepa (13 → 15)
|
|
2046
|
+
grep -q "tools.length === 15" "$F" || miss="$miss tool-count-assertion-stale"
|
|
2042
2047
|
# Graceful skip when dist absent (so the script is smoke-runnable pre-build)
|
|
2043
2048
|
grep -q "SKIPPED" "$F" || miss="$miss no-skip-doc"
|
|
2044
2049
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
@@ -74,7 +74,7 @@ async function main() {
|
|
|
74
74
|
// ──────────────────────────────────────────────────────────────────
|
|
75
75
|
console.log('Phase 1 — module shape');
|
|
76
76
|
assert(Array.isArray(tools), 'metaharnessTools is an array');
|
|
77
|
-
assert(tools.length ===
|
|
77
|
+
assert(tools.length === 15, `15 tools registered (got ${tools.length})`);
|
|
78
78
|
|
|
79
79
|
const expectedNames = new Set([
|
|
80
80
|
'metaharness_score',
|
|
@@ -92,8 +92,12 @@ async function main() {
|
|
|
92
92
|
'metaharness_bench',
|
|
93
93
|
'metaharness_evolve',
|
|
94
94
|
'metaharness_security_bench',
|
|
95
|
-
// @metaharness/redblue@~0.1.
|
|
95
|
+
// @metaharness/redblue@~0.1.4 — adversarial red/blue LLM testing
|
|
96
96
|
'metaharness_redblue',
|
|
97
|
+
// metaharness@0.3.0 — upstream ADR-235 GEPA learning run
|
|
98
|
+
'metaharness_learn',
|
|
99
|
+
// @metaharness/darwin@0.8.0 — GEPA library surface (genome ops)
|
|
100
|
+
'metaharness_gepa',
|
|
97
101
|
]);
|
|
98
102
|
const actualNames = new Set(tools.map((t) => t.name));
|
|
99
103
|
for (const name of expectedNames) {
|
|
@@ -153,6 +157,16 @@ async function main() {
|
|
|
153
157
|
// calls. Count=1 keeps cold-cache npx fetch the dominant cost.
|
|
154
158
|
input = { subcommand: 'attack', family: 'prompt', count: 1 };
|
|
155
159
|
}
|
|
160
|
+
if (tool.name === 'metaharness_learn') {
|
|
161
|
+
// No repo checkout in CI → structured {status:"checkout-required"}
|
|
162
|
+
// exit-0 path. $0: without run=true upstream never spends anyway.
|
|
163
|
+
input = {};
|
|
164
|
+
}
|
|
165
|
+
if (tool.name === 'metaharness_gepa') {
|
|
166
|
+
// op is required; `genome` loads + validates the SHIPPED cand-6
|
|
167
|
+
// genome — pure-local library call once darwin is cached.
|
|
168
|
+
input = { op: 'genome' };
|
|
169
|
+
}
|
|
156
170
|
|
|
157
171
|
// iter 124 → 130 — timeouts have crept up as CI cold-cache npx
|
|
158
172
|
// warmup costs got measured. Final budgets:
|
|
@@ -167,9 +181,14 @@ async function main() {
|
|
|
167
181
|
|| tool.name === 'metaharness_oia_audit'
|
|
168
182
|
|| tool.name === 'metaharness_audit_list'
|
|
169
183
|
// redblue: `attack prompt --count 1` is preview-only (no model
|
|
170
|
-
// calls) but the cold-cache `npx -y @metaharness/redblue@~0.1.
|
|
184
|
+
// calls) but the cold-cache `npx -y @metaharness/redblue@~0.1.4`
|
|
171
185
|
// fetch can take 30-60s. 180s gives 3x headroom.
|
|
172
|
-
|| tool.name === 'metaharness_redblue'
|
|
186
|
+
|| tool.name === 'metaharness_redblue'
|
|
187
|
+
// learn: cold-cache `npx -y metaharness@latest` fetch dominates.
|
|
188
|
+
// gepa: one-time `npm install --prefix ~/.ruflo/darwin-cache-*`
|
|
189
|
+
// fallback install can take 30-60s on cold cache.
|
|
190
|
+
|| tool.name === 'metaharness_learn'
|
|
191
|
+
|| tool.name === 'metaharness_gepa';
|
|
173
192
|
const timeoutMs = isChainTool ? 180_000 : 60_000;
|
|
174
193
|
const handlerPromise = tool.handler(input);
|
|
175
194
|
const timeoutPromise = new Promise((_, reject) =>
|
|
@@ -444,7 +463,7 @@ async function main() {
|
|
|
444
463
|
for (const f of failures) console.log(` - ${f}`);
|
|
445
464
|
process.exit(1);
|
|
446
465
|
}
|
|
447
|
-
console.log('\n✓ All
|
|
466
|
+
console.log('\n✓ All 15 MCP tools satisfy the runtime contract.');
|
|
448
467
|
}
|
|
449
468
|
|
|
450
469
|
main().catch((e) => {
|
|
@@ -35,7 +35,7 @@ Implementation: [`scripts/evolve.mjs`](../../scripts/evolve.mjs).
|
|
|
35
35
|
≤ 20, `--concurrency` ≤ 8, sandbox/selection/mutator are known values).
|
|
36
36
|
2. Without `--confirm`: print plan + exit 0 (mirrors `harness-mint` safety
|
|
37
37
|
convention; defense in depth over the upstream `safety.ts` checks).
|
|
38
|
-
3. With `--confirm`: shell to `npx -y @metaharness/darwin@~0.
|
|
38
|
+
3. With `--confirm`: shell to `npx -y @metaharness/darwin@~0.8.0 metaharness-darwin evolve <repo> ...`
|
|
39
39
|
via the shared `_darwin.mjs` async helper. Per-generation progress is
|
|
40
40
|
forwarded to stderr; final champion JSON is captured from stdout.
|
|
41
41
|
4. Compute timeout from `generations × children × per-variant` (per-variant
|