@claude-flow/cli 3.12.4 → 3.13.0

package/plugins/ruflo-metaharness/scripts/security-bench.mjs

@@ -0,0 +1,174 @@
+#!/usr/bin/env node
+// security-bench.mjs — wrapper around `metaharness-darwin security bench`.
+//
+// "Darwin Shield" — upstream's own ADR-155 — evolves a champion harness
+// on a 10-vuln/9-decoy corpus and measures it against four baselines
+// (B0 static-only, B1 LLM single-pass, B2 fixed agent, B3 Darwin champion).
+// The CHAMPION reaches TPR=1, FPR=0; the eight acceptance gates verify
+// reproducibility, statistical significance, and unsafe-output=0.
+//
+// CONNECTION TO RUFLO ADR-155
+// ===========================
+// ruflo's ADR-155 (#2417) proposes a nightly self-learning security harness
+// with three learning loops (per-dimension confidence weighting, severity
+// calibration, auto-fix bid). The upstream Darwin Shield is the closest
+// reference implementation — same shape, different scope (evolves a
+// security-detection harness vs evaluates findings; both grade by realized
+// TPR/FPR vs ground-truth corpus). Running `security bench` periodically
+// gives us the empirical baseline that ruflo's Phase 2 loop A needs before
+// training: if Darwin Shield converges on a known-good corpus, the loop A
+// gradient signal is sound; if it doesn't, the corpus / sandbox is the
+// gap, not the learning algorithm.
+//
+// USAGE
+//   node scripts/security-bench.mjs                              # default population=2 cycles=1
+//   node scripts/security-bench.mjs --population 4 --cycles 3    # deeper run
+//   node scripts/security-bench.mjs --population 4 --cycles 3 --alert-on-fail
+//
+// EXIT CODES
+//   0  bench passed all gates (or degraded — Darwin not available)
+//   1  --alert-on-fail and any acceptance gate failed
+//   2  config error or bench infrastructure failure
+
+import { runDarwinAsync, emitDarwinDegradedJsonAndExit } from './_darwin.mjs';
+
+const ARGS = (() => {
+  const a = {
+    population: 2,
+    cycles: 1,
+    seed: null,
+    alertOnFail: false,
+    format: 'json',
+    timeoutMs: null,
+  };
+  for (let i = 2; i < process.argv.length; i++) {
+    const v = process.argv[i];
+    if (v === '--population') a.population = parseInt(process.argv[++i], 10);
+    else if (v === '--cycles') a.cycles = parseInt(process.argv[++i], 10);
+    else if (v === '--seed') a.seed = parseInt(process.argv[++i], 10);
+    else if (v === '--alert-on-fail') a.alertOnFail = true;
+    else if (v === '--format') a.format = process.argv[++i];
+    else if (v === '--timeout-ms') a.timeoutMs = parseInt(process.argv[++i], 10);
+  }
+  return a;
+})();
+
+function safetyChecks() {
+  if (ARGS.population < 1 || ARGS.population > 20) {
+    console.error('security-bench: --population must be 1..20 (ruflo cap)');
+    process.exit(2);
+  }
+  if (ARGS.cycles < 1 || ARGS.cycles > 100) {
+    console.error('security-bench: --cycles must be 1..100 (ruflo cap)');
+    process.exit(2);
+  }
+}
+
+function defaultTimeoutMs() {
+  // Bench corpus has 10 vulns + 9 decoys = 19 evaluations per cycle.
+  // Each evaluation runs the candidate detector + scores patches.
+  // Rough budget: ~3s per evaluation × population × cycles + 30s overhead.
+  return Math.max(60_000, 3_000 * 19 * ARGS.population * ARGS.cycles + 30_000);
+}
+
+// Parse the markdown report that `security bench` emits. The header
+// "Overall: ✅ PASS" or "❌ FAIL" is the rolled-up gate. We also extract
+// each acceptance gate's pass/fail line.
+function parseSecurityBenchMarkdown(stdout) {
+  const overallMatch = /\*\*Overall:\s*([✅❌])\s*(PASS|FAIL)\*\*/i.exec(stdout);
+  const overall = overallMatch ? { ok: overallMatch[2].toUpperCase() === 'PASS', icon: overallMatch[1] } : null;
+
+  const gates = [];
+  // Match lines like: "- ✅ **TPR improvement ≥ 25% vs fixed harness** — +150%"
+  const gateRx = /^-\s+([✅❌])\s+\*\*(.+?)\*\*\s*[—-]\s*(.+)$/gm;
+  let m;
+  while ((m = gateRx.exec(stdout)) !== null) {
+    gates.push({
+      ok: m[1] === '✅',
+      criterion: m[2].trim(),
+      measured: m[3].trim(),
+    });
+  }
+
+  // Extract the baselines-vs-champion table — useful for diff over time.
+  const baselines = [];
+  const tableRx = /^\|\s*B[0-3]\s+([^|]+?)\s*\|\s*([\d.]+)\s*\|\s*([\d.]+)\s*\|\s*([\d.]+)\s*\|\s*([\d.]+)\s*\|\s*([\d.]+)\s*\|\s*(\d+)\s*\|\s*(.+?)\s*\|/gm;
+  while ((m = tableRx.exec(stdout)) !== null) {
+    baselines.push({
+      harness: m[1].trim(),
+      fitness: parseFloat(m[2]),
+      tpr: parseFloat(m[3]),
+      fpr: parseFloat(m[4]),
+      patchPass: parseFloat(m[5]),
+      repro: parseFloat(m[6]),
+      unsafe: parseInt(m[7], 10),
+      cost: m[8].trim(),
+    });
+  }
+
+  return { overall, gates, baselines };
+}
+
+async function main() {
+  safetyChecks();
+
+  const cliArgs = ['security', 'bench',
+    '--population', String(ARGS.population),
+    '--cycles', String(ARGS.cycles),
+  ];
+  if (ARGS.seed != null) cliArgs.push('--seed', String(ARGS.seed));
+
+  const r = await runDarwinAsync(cliArgs, {
+    timeoutMs: ARGS.timeoutMs ?? defaultTimeoutMs(),
+    json: false, // bench output is markdown, not JSON
+    onProgress: (line) => { if (line.trim()) process.stderr.write(`[security-bench] ${line}\n`); },
+  });
+
+  if (r.degraded) {
+    emitDarwinDegradedJsonAndExit(r.reason);
+    return;
+  }
+
+  if (r.exitCode !== 0 && r.exitCode !== 1) {
+    // Exit 1 may be the bench's own "gates failed" signal; treat as data.
+    // Anything else is infrastructure failure.
+    const payload = {
+      success: false,
+      data: { exitCode: r.exitCode, stderrTail: r.stderr.slice(-400) },
+      generatedAt: new Date().toISOString(),
+    };
+    console.log(JSON.stringify(payload, null, 2));
+    process.exit(2);
+  }
+
+  const parsed = parseSecurityBenchMarkdown(r.stdout);
+  const gatesPassed = parsed.gates.filter((g) => g.ok).length;
+  const gatesFailed = parsed.gates.length - gatesPassed;
+
+  const payload = {
+    success: true,
+    data: {
+      overall: parsed.overall,
+      gates: {
+        total: parsed.gates.length,
+        passed: gatesPassed,
+        failed: gatesFailed,
+        details: parsed.gates,
+      },
+      baselines: parsed.baselines,
+      rawMarkdown: r.stdout,
+      shape: { population: ARGS.population, cycles: ARGS.cycles, seed: ARGS.seed },
+      durationMs: r.durationMs,
+    },
+    generatedAt: new Date().toISOString(),
+  };
+
+  console.log(JSON.stringify(payload, null, 2));
+  if (ARGS.alertOnFail && parsed.overall && !parsed.overall.ok) process.exit(1);
+  process.exit(0);
+}
+
+main().catch((e) => {
+  console.error(`security-bench: unexpected failure: ${e?.message ?? e}`);
+  process.exit(2);
+});

package/plugins/ruflo-metaharness/scripts/test-graceful-degradation.mjs

@@ -111,6 +111,14 @@ function main() {
     { name: 'mcp-scan',     args: ['--format', 'json'] },
     { name: 'threat-model', args: ['--format', 'json'] },
     { name: 'oia-audit',    args: ['--dry-run', '--format', 'json'] },
+    // ADR-153 — darwin scripts (separate optional dep @metaharness/darwin).
+    // evolve.mjs without --confirm short-circuits to a dry-run plan BEFORE
+    // any subprocess call, so it's not exercising the degraded path. Pass
+    // --confirm with --sandbox mock + minimal shape so we hit the subprocess
+    // path quickly and verify the {degraded: true} emit on registry-unreachable.
+    { name: 'evolve',       args: ['--repo', '.', '--confirm', '--sandbox', 'mock', '--generations', '1', '--children', '1', '--concurrency', '1', '--timeout-ms', '60000'] },
+    { name: 'security-bench', args: ['--population', '1', '--cycles', '1', '--timeout-ms', '60000'] },
+    { name: 'bench',          args: ['--op', 'verify', '--suite', '/dev/null'] },
   ];
 
   for (const s of skills) {

package/plugins/ruflo-metaharness/skills/harness-bench/SKILL.md

@@ -0,0 +1,64 @@
+---
+name: harness-bench
+description: Manage `@metaharness/darwin` bench suites — `bench create <repo>` scaffolds a JSON suite from a repo's test corpus; `bench verify <suite.json>` checks suite well-formedness. Bench suites are the fixed evaluation corpora that `harness-evolve --bench <suite.json>` scores variants against, decoupling evolution from the repo's natural tests. Degrades gracefully when @metaharness/darwin is absent.
+argument-hint: "--op create --repo <path> [--out <path>]  |  --op verify --suite <path>"
+allowed-tools: Bash
+---
+
+Surfaces `metaharness-darwin bench <create|verify>` — the supporting verb
+for `harness-evolve --bench`. Use when you want evolution scored against a
+fixed corpus (independent of `npm test`) so champion fitness is comparable
+across commits or across forks of the same harness.
+
+## When to use
+
+- Setting up a new evolution pipeline for a repo whose `npm test` is
+  flaky, slow, or undersized — scaffold a deterministic bench suite once,
+  then evolve against it repeatedly.
+- CI: `bench verify` the checked-in suite on every PR that touches it
+  (cheap; ~5s).
+- Forking a harness to a new domain: copy and edit the suite to retarget
+  the evaluation without losing comparability to the parent.
+
+## Algorithm
+
+Implementation: [`scripts/bench.mjs`](../../scripts/bench.mjs).
+
+### `--op create`
+1. Resolve `--repo` path; reject if missing.
+2. Shell to `metaharness-darwin bench create <repo> [--out <suite.json>]`.
+3. Default output path: `<repo>/.metaharness/bench/suite.json` (chosen by upstream).
+4. Suite shape (per upstream): array of `{ input, expectedOutput, weight }` tasks
+   derived from existing test cases.
+
+### `--op verify`
+1. Resolve `--suite` path; reject if missing.
+2. Shell to `metaharness-darwin bench verify <suite.json>`.
+3. Exit 1 if any task malformed (upstream's signal).
+
+## Output shape
+
+```json
+{
+  "success": true,
+  "data": {
+    "op": "verify",
+    "taskCount": 42,
+    "wellFormed": true,
+    "durationMs": 870
+  }
+}
+```
+
+## Exit codes
+
+| Code | Meaning |
+|---|---|
+| 0 | OK (or degraded — Darwin absent) |
+| 1 | `--op verify` and suite malformed |
+| 2 | Config error or upstream invocation failure |
+
+## Graceful degradation
+
+When `@metaharness/darwin` is absent, emits the standard `{degraded: true,
+reason: 'metaharness-darwin-not-available'}` payload and exits 0.

package/plugins/ruflo-metaharness/skills/harness-evolve/SKILL.md

@@ -0,0 +1,92 @@
+---
+name: harness-evolve
+description: Run `@metaharness/darwin evolve <repo>` to mutate a harness's seven policy surfaces (planner/contextBuilder/reviewer/retryPolicy/toolPolicy/memoryPolicy/scorePolicy), sandbox-score each variant, and promote only measured wins. The model is frozen; the harness evolves. Closes the loop ADR-150 opens (score+genome describe; evolve changes). Degrades gracefully when @metaharness/darwin is absent (ADR-150 + ADR-153 architectural constraints).
+argument-hint: "--repo <path> [--generations 3] [--children 3] [--concurrency 2] [--sandbox real|mock|agent] [--selection pareto|quality-diversity|...] [--mutator deterministic|ruvllm] [--confirm]"
+allowed-tools: Bash
+---
+
+Surfaces the upstream `metaharness-darwin evolve` CLI as a ruflo skill. The
+**write** layer that pairs with ADR-150's read layer (score / genome /
+mcp-scan / threat-model / oia-audit). Use when you have a harness whose
+readiness scores are flat and you want to discover *which* surface mutation
+moves them — without retraining the foundation model.
+
+## When to use
+
+- A `harness-score` result is below target and you don't know which policy
+  surface is responsible.
+- You're seeding a harness for a new vertical and want to find a good
+  starting configuration empirically rather than hand-tuning.
+- You're comparing your hand-tuned harness against an evolved baseline
+  (treat darwin's champion as the strawman).
+
+## When NOT to use
+
+- For continuous background optimization. Darwin Mode is human-initiated.
+  Wire it into CI for one-shot exploration, not for autonomous self-modification.
+- For ruflo itself in CI. ADR-153 §5 explicitly rejects auto-evolving ruflo
+  — the CI gate verifies graceful degradation, not convergence.
+
+## Algorithm
+
+Implementation: [`scripts/evolve.mjs`](../../scripts/evolve.mjs).
+
+1. Validate args (`--repo` exists, caps on `--generations` ≤ 50, `--children`
+   ≤ 20, `--concurrency` ≤ 8, sandbox/selection/mutator are known values).
+2. Without `--confirm`: print plan + exit 0 (mirrors `harness-mint` safety
+   convention; defense in depth over the upstream `safety.ts` checks).
+3. With `--confirm`: shell to `npx -y @metaharness/darwin@~0.3.1 metaharness-darwin evolve <repo> ...`
+   via the shared `_darwin.mjs` async helper. Per-generation progress is
+   forwarded to stderr; final champion JSON is captured from stdout.
+4. Compute timeout from `generations × children × per-variant` (per-variant
+   ≈ 60s real, ≈ 2s mock). Caller may override with `--timeout-ms`.
+5. Honor upstream exit code 99 — propagate as "safety-disqualified", do not
+   remap. This is a designed-in tripwire (a variant tripped `inspectVariant`
+   for secrets / shell-out / network / dynamic-eval). See ADR-153 §"Safety model".
+6. Optional `--alert-on-no-improvement`: exit 1 when champion ≤ parent.
+
+## The seven mutation surfaces
+
+| Surface | What it owns |
+|---|---|
+| `planner` | task decomposition / step ordering |
+| `contextBuilder` | what gets fed into the prompt |
+| `reviewer` | self-critique / output verification |
+| `retryPolicy` | when + how to retry on failure |
+| `toolPolicy` | which tools the agent may use, under which conditions |
+| `memoryPolicy` | what to persist, recall, forget |
+| `scorePolicy` | how the agent grades its own output |
+
+One mutation per variant. Multi-surface mutations are not allowed (causal
+attribution stays clean).
+
+## Output
+
+Reports land under `<repo>/.metaharness/`:
+
+```
+.metaharness/
+  archive.json         # full lineage tree (sampling next gen draws from this)
+  lineage.json         # parent→child edges only
+  variants/<id>/       # per-variant code (kept for audit)
+  runs/<id>/           # per-variant sandbox test output
+  reports/winner.json  # final champion + score delta vs parent
+```
+
+Skill stdout = JSON `{success, data: {champion, plan, durationMs, improved}}`.
+
+## Exit codes
+
+| Code | Meaning |
+|---|---|
+| 0  | Evolved OK, or dry-run, or degraded (Darwin absent) |
+| 1  | `--alert-on-no-improvement` and champion did not beat parent |
+| 2  | Config error or evolution infrastructure failure |
+| 99 | Upstream "safety-disqualified" (PROPAGATED, not remapped) |
+
+## Graceful degradation (ADR-150 constraint 3 + ADR-153)
+
+When `@metaharness/darwin` is not installed, the script emits
+`{degraded: true, reason: 'metaharness-darwin-not-available', hint: ...}`
+and exits 0. ruflo continues to function. CI's
+`no-metaharness-smoke.yml`-style job asserts this path.

package/plugins/ruflo-metaharness/skills/harness-security-bench/SKILL.md

@@ -0,0 +1,101 @@
+---
+name: harness-security-bench
+description: Run `@metaharness/darwin security bench` (upstream "Darwin Shield" / ADR-155) — evolves a champion security-detection harness against a 10-vuln / 9-decoy corpus and grades it on TPR/FPR/patch-pass/repro/unsafe vs four baselines (B0 static, B1 LLM-single-pass, B2 fixed-agent, B3 Darwin-champion). Closest reference implementation for ruflo's own ADR-155 nightly self-learning security harness (PR #2417). Degrades gracefully when @metaharness/darwin is absent.
+argument-hint: "[--population 2] [--cycles 1] [--seed N] [--alert-on-fail]"
+allowed-tools: Bash
+---
+
+Surfaces the upstream `metaharness-darwin security bench` command. **This is
+the upstream's own ADR-155 — Darwin Shield — and is the closest reference
+implementation for ruflo's nightly self-learning security harness ([#2417](https://github.com/ruvnet/ruflo/pull/2417)).**
+
+## Why this matters for ruflo's ADR-155
+
+ruflo's ADR-155 proposes three learning loops (per-dimension confidence,
+severity calibration, auto-fix bid). Loop A trains on accumulated
+`(finding, dimension, human_outcome)` tuples — but the gradient signal is
+only sound if the underlying detection mechanism converges on a known-good
+corpus. Darwin Shield evolves exactly that mechanism on a 10-vuln/9-decoy
+ground-truth set. Running this nightly gives us:
+
+- **Empirical floor:** if Darwin Shield's champion can't reach
+  TPR=1/FPR=0 on the bench corpus, our Loop A's reward signal is noise.
+- **Drift detection:** week-over-week champion fitness deltas surface
+  when the security landscape (or our mutator policy) shifts.
+- **Baseline diversity:** the 4 baselines (B0–B3) give us 4 anchor
+  points to weight per-dimension confidence against.
+
+## Algorithm
+
+Implementation: [`scripts/security-bench.mjs`](../../scripts/security-bench.mjs).
+
+1. Shell to `npx -y @metaharness/darwin@~0.3.1 metaharness-darwin security bench --population N --cycles N [--seed S]`.
+2. Default timeout = `3s × 19 evaluations × population × cycles + 30s overhead`.
+   At default `--population 2 --cycles 1` ≈ 144s; at `--population 4 --cycles 3` ≈ 12 min.
+3. Parse the markdown report — overall PASS/FAIL plus per-gate
+   pass/fail rows (gate examples: "TPR improvement ≥ 25% vs fixed",
+   "FPR reduction ≥ 40%", "Patch-test pass rate ≥ 80%", "Reproduction
+   success ≥ 90%", "Unsafe outputs = 0", "Cost increase ≤ 2× fixed",
+   "Beyond SOTA: champion statistically beats previous champion",
+   "Compounding: false-positive repeat-rate drop ≥ 35%").
+4. Parse the baselines-vs-champion table (4 rows: fitness/TPR/FPR/patchPass/
+   repro/unsafe/cost per harness).
+5. Emit structured JSON. With `--alert-on-fail`, exit 1 when overall = FAIL.
+
+## Output shape
+
+```json
+{
+  "success": true,
+  "data": {
+    "overall": { "ok": true, "icon": "✅" },
+    "gates": {
+      "total": 11,
+      "passed": 11,
+      "failed": 0,
+      "details": [{ "ok": true, "criterion": "TPR improvement ≥ 25% vs fixed harness", "measured": "+150% (B2 0.4 → B3 1)" }, ...]
+    },
+    "baselines": [
+      { "harness": "static-only", "fitness": 0.5665, "tpr": 0.3, "fpr": 1, "unsafe": 0, ... },
+      { "harness": "LLM single-pass", "fitness": 0.1365, ... },
+      { "harness": "fixed agent", "fitness": 0.598, ... },
+      { "harness": "Darwin champion", "fitness": 0.93275, "tpr": 1, "fpr": 0, ... }
+    ],
+    "rawMarkdown": "...",
+    "shape": { "population": 2, "cycles": 1, "seed": null },
+    "durationMs": 142000
+  }
+}
+```
+
+## Wiring into ADR-155 nightly harness
+
+The ADR-155 nightly workflow (per #2418 task `W1.5`) will spawn this as
+one of the active-pentest dimension's calls — its results become a
+trajectory record:
+
+```jsonc
+{
+  "dimension": "mcp-pentest",
+  "subdimension": "darwin-shield-bench",
+  "champion_fitness": 0.93275,
+  "champion_tpr": 1, "champion_fpr": 0,
+  "gates_passed": 11, "gates_failed": 0,
+  "shape": { "population": 4, "cycles": 3 }
+}
+```
+
+Loop A learns: if `darwin-shield-bench` consistently passes on the seeded
+corpus, weight findings caught only by `mcp-pentest` higher.
+
+## Exit codes
+
+| Code | Meaning |
+|---|---|
+| 0 | Bench ran (overall PASS or FAIL — distinguish via JSON `overall.ok`), or degraded |
+| 1 | `--alert-on-fail` and `overall.ok === false` |
+| 2 | Config error or upstream infrastructure failure |
+
+## Graceful degradation
+
+When `@metaharness/darwin` is absent, emits `{degraded: true, reason: 'metaharness-darwin-not-available'}` and exits 0.