@claude-flow/cli 3.32.9 → 3.32.10

package/plugins/ruflo-metaharness/scripts/test-graceful-degradation.mjs

@@ -1,165 +1,165 @@
-#!/usr/bin/env node
-// test-graceful-degradation.mjs — runtime drill for ADR-150 architectural
-// constraint rule #3 (graceful degradation).
-//
-// CI runs this via .github/workflows/no-metaharness-smoke.yml; this
-// script lets developers reproduce the same drill locally for fast
-// iteration (no GitHub Actions roundtrip).
-//
-// STRATEGY
-// Point `npm_config_registry` at an unresolvable host, then invoke every
-// metaharness-CLI-dependent skill. Each MUST:
-//   (a) exit 0 — never propagate a failure code from a missing optional dep
-//   (b) emit JSON containing `"degraded": true` somewhere in its output
-//
-// The 8 covered skills (all rely on `npx metaharness` / `npx -p
-// metaharness harness`):
-//   score / genome / mcp-scan / threat-model / oia-audit /
-//   audit-list / audit-trend / mint
-//
-// (audit-list + audit-trend talk to `npx @claude-flow/cli memory ...`
-//  rather than metaharness — they're tested separately for the
-//  "no records in namespace" graceful path.)
-//
-// USAGE
-//   node scripts/test-graceful-degradation.mjs                 # default
-//   node scripts/test-graceful-degradation.mjs --keep-fixtures # for debugging
-//
-// EXIT CODES
-//   0  all skills gracefully degraded
-//   1  at least one skill failed the contract (exit != 0 or no degraded:true)
-//   2  setup error
-
-import { spawnSync } from 'node:child_process';
-import { mkdtempSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-const SCRIPTS_DIR = dirname(fileURLToPath(import.meta.url));
-
-// Since the versioned-cache consolidation (_invoke.mjs), a warm
-// ~/.ruflo/<pkg>-cache-<pin> or a locally installed metaharness would satisfy
-// resolution WITHOUT touching the (unreachable) registry — making this drill
-// vacuous on developer machines. Point the cache base at an empty temp dir
-// and disable local walk-up resolution so the install path (and therefore
-// the degraded contract) is actually exercised.
-const EMPTY_CACHE_BASE = mkdtempSync(join(tmpdir(), 'ruflo-degradation-drill-'));
-
-const ARGS = (() => {
-  const a = { keep: false };
-  for (let i = 2; i < process.argv.length; i++) {
-    if (process.argv[i] === '--keep-fixtures') a.keep = true;
-  }
-  return a;
-})();
-
-const UNREACHABLE_REGISTRY = 'https://no-such-registry-9c8c43.example.invalid/';
-
-let passed = 0, failed = 0;
-const failures = [];
-
-function assert(cond, label) {
-  console.log(`  ${cond ? '✓' : '✗'} ${label}`);
-  if (cond) passed++;
-  else { failures.push(label); failed++; }
-}
-
-function runWithUnreachableRegistry(scriptName, extraArgs) {
-  const r = spawnSync('node', [join(SCRIPTS_DIR, scriptName), ...extraArgs], {
-    env: {
-      ...process.env,
-      npm_config_registry: UNREACHABLE_REGISTRY,
-      NPM_CONFIG_REGISTRY: UNREACHABLE_REGISTRY,
-      RUFLO_METAHARNESS_CACHE_BASE: EMPTY_CACHE_BASE,
-      RUFLO_METAHARNESS_SKIP_LOCAL: '1',
-    },
-    stdio: ['ignore', 'pipe', 'pipe'],
-    encoding: 'utf-8',
-    // 180s per subprocess — long enough for npx's DNS-retry + 60s
-    // subprocess timeout inside _harness.mjs to elapse. Smaller
-    // timeouts here truncate the subprocess BEFORE it reaches its
-    // own degraded-payload emit, which made earlier iterations of
-    // this drill report false negatives (exit 1 from harness kill,
-    // not from the script).
-    timeout: 180_000,
-  });
-  return {
-    // r.status is null when the subprocess was killed by our timeout.
-    // Distinguish that case explicitly so callers can see "drill
-    // harness killed it" vs "script exited non-zero".
-    exitCode: r.status,
-    stdout: r.stdout || '',
-    stderr: r.stderr || '',
-    killedByTimeout: r.status === null,
-  };
-}
-
-function main() {
-  console.log('# test-graceful-degradation\n');
-  console.log(`Registry: ${UNREACHABLE_REGISTRY}`);
-  console.log('Each skill must exit 0 AND emit "degraded": true in JSON.\n');
-
-  // The 5 skills that directly invoke metaharness/harness binaries.
-  // (audit-list / audit-trend talk to claude-flow's memory layer, not
-  // metaharness; they have their own empty-namespace graceful paths
-  // tested separately. mint requires a --name argv to even start.)
-  // iter 55 — extending this list discovered 3 latent gaps:
-  //   - oia-audit can exceed 180s in the unreachable-registry case
-  //     (composes 5 subprocesses, each retries DNS)
-  //   - mint's degraded payload doesn't currently include the
-  //     literal "degraded": true marker
-  //   - drift-from-history exits 2 (config error: no history) rather
-  //     than 3 (test-cannot-run) when there are no audit records yet
-  // These are real bugs filed for follow-up; the workflow-level drill
-  // (`.github/workflows/no-metaharness-smoke.yml`) covers them in a
-  // fresh CI environment where DNS failure is faster, but they're
-  // unreliable to assert locally. Keeping the local drill at the 5
-  // skills it reliably tests until the underlying graceful-degradation
-  // contracts are tightened.
-  const skills = [
-    { name: 'score',        args: ['--format', 'json'] },
-    { name: 'genome',       args: ['--format', 'json'] },
-    { name: 'mcp-scan',     args: ['--format', 'json'] },
-    { name: 'threat-model', args: ['--format', 'json'] },
-    { name: 'oia-audit',    args: ['--dry-run', '--format', 'json'] },
-    // ADR-153 — darwin scripts (separate optional dep @metaharness/darwin).
-    // evolve.mjs without --confirm short-circuits to a dry-run plan BEFORE
-    // any subprocess call, so it's not exercising the degraded path. Pass
-    // --confirm with --sandbox mock + minimal shape so we hit the subprocess
-    // path quickly and verify the {degraded: true} emit on registry-unreachable.
-    { name: 'evolve',       args: ['--repo', '.', '--confirm', '--sandbox', 'mock', '--generations', '1', '--children', '1', '--concurrency', '1', '--timeout-ms', '60000'] },
-    { name: 'security-bench', args: ['--population', '1', '--cycles', '1', '--timeout-ms', '60000'] },
-    { name: 'bench',          args: ['--op', 'verify', '--suite', '/dev/null'] },
-  ];
-
-  for (const s of skills) {
-    console.log(`-- ${s.name} --`);
-    const r = runWithUnreachableRegistry(`${s.name}.mjs`, s.args);
-    if (process.env.DEBUG || r.killedByTimeout || r.exitCode !== 0) {
-      // Surface the failure context: drill timeouts and unexpected exits
-      // are both worth seeing, otherwise debugging is blind.
-      console.log(`  stdout (last 400): ${r.stdout.slice(-400)}`);
-      console.log(`  stderr (last 300): ${r.stderr.slice(-300)}`);
-      if (r.killedByTimeout) console.log(`  (killed by drill 180s timeout)`);
-    }
-    const acceptable = s.acceptableExits ?? [0];
-    assert(acceptable.includes(r.exitCode) || (acceptable.length === 1 && r.exitCode === 0),
-      `${s.name} exit code in {${acceptable.join(',')}} (got ${r.killedByTimeout ? 'timeout' : r.exitCode})`);
-    assert(/"degraded"\s*:\s*true/.test(r.stdout), `${s.name} emits "degraded": true`);
-  }
-
-  if (!ARGS.keep) {
-    try { rmSync(EMPTY_CACHE_BASE, { recursive: true, force: true }); } catch { /* best-effort */ }
-  }
-
-  console.log(`\n${passed} passed, ${failed} failed`);
-  if (failed > 0) {
-    console.log('\nFailures:');
-    for (const f of failures) console.log(`  - ${f}`);
-    process.exit(1);
-  }
-  console.log('\n✓ All skills satisfy ADR-150 rule #3 (graceful degradation).');
-}
-
-main();
+#!/usr/bin/env node
+// test-graceful-degradation.mjs — runtime drill for ADR-150 architectural
+// constraint rule #3 (graceful degradation).
+//
+// CI runs this via .github/workflows/no-metaharness-smoke.yml; this
+// script lets developers reproduce the same drill locally for fast
+// iteration (no GitHub Actions roundtrip).
+//
+// STRATEGY
+// Point `npm_config_registry` at an unresolvable host, then invoke every
+// metaharness-CLI-dependent skill. Each MUST:
+//   (a) exit 0 — never propagate a failure code from a missing optional dep
+//   (b) emit JSON containing `"degraded": true` somewhere in its output
+//
+// The 8 covered skills (all rely on `npx metaharness` / `npx -p
+// metaharness harness`):
+//   score / genome / mcp-scan / threat-model / oia-audit /
+//   audit-list / audit-trend / mint
+//
+// (audit-list + audit-trend talk to `npx @claude-flow/cli memory ...`
+//  rather than metaharness — they're tested separately for the
+//  "no records in namespace" graceful path.)
+//
+// USAGE
+//   node scripts/test-graceful-degradation.mjs                 # default
+//   node scripts/test-graceful-degradation.mjs --keep-fixtures # for debugging
+//
+// EXIT CODES
+//   0  all skills gracefully degraded
+//   1  at least one skill failed the contract (exit != 0 or no degraded:true)
+//   2  setup error
+
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const SCRIPTS_DIR = dirname(fileURLToPath(import.meta.url));
+
+// Since the versioned-cache consolidation (_invoke.mjs), a warm
+// ~/.ruflo/<pkg>-cache-<pin> or a locally installed metaharness would satisfy
+// resolution WITHOUT touching the (unreachable) registry — making this drill
+// vacuous on developer machines. Point the cache base at an empty temp dir
+// and disable local walk-up resolution so the install path (and therefore
+// the degraded contract) is actually exercised.
+const EMPTY_CACHE_BASE = mkdtempSync(join(tmpdir(), 'ruflo-degradation-drill-'));
+
+const ARGS = (() => {
+  const a = { keep: false };
+  for (let i = 2; i < process.argv.length; i++) {
+    if (process.argv[i] === '--keep-fixtures') a.keep = true;
+  }
+  return a;
+})();
+
+const UNREACHABLE_REGISTRY = 'https://no-such-registry-9c8c43.example.invalid/';
+
+let passed = 0, failed = 0;
+const failures = [];
+
+function assert(cond, label) {
+  console.log(`  ${cond ? '✓' : '✗'} ${label}`);
+  if (cond) passed++;
+  else { failures.push(label); failed++; }
+}
+
+function runWithUnreachableRegistry(scriptName, extraArgs) {
+  const r = spawnSync('node', [join(SCRIPTS_DIR, scriptName), ...extraArgs], {
+    env: {
+      ...process.env,
+      npm_config_registry: UNREACHABLE_REGISTRY,
+      NPM_CONFIG_REGISTRY: UNREACHABLE_REGISTRY,
+      RUFLO_METAHARNESS_CACHE_BASE: EMPTY_CACHE_BASE,
+      RUFLO_METAHARNESS_SKIP_LOCAL: '1',
+    },
+    stdio: ['ignore', 'pipe', 'pipe'],
+    encoding: 'utf-8',
+    // 180s per subprocess — long enough for npx's DNS-retry + 60s
+    // subprocess timeout inside _harness.mjs to elapse. Smaller
+    // timeouts here truncate the subprocess BEFORE it reaches its
+    // own degraded-payload emit, which made earlier iterations of
+    // this drill report false negatives (exit 1 from harness kill,
+    // not from the script).
+    timeout: 180_000,
+  });
+  return {
+    // r.status is null when the subprocess was killed by our timeout.
+    // Distinguish that case explicitly so callers can see "drill
+    // harness killed it" vs "script exited non-zero".
+    exitCode: r.status,
+    stdout: r.stdout || '',
+    stderr: r.stderr || '',
+    killedByTimeout: r.status === null,
+  };
+}
+
+function main() {
+  console.log('# test-graceful-degradation\n');
+  console.log(`Registry: ${UNREACHABLE_REGISTRY}`);
+  console.log('Each skill must exit 0 AND emit "degraded": true in JSON.\n');
+
+  // The 5 skills that directly invoke metaharness/harness binaries.
+  // (audit-list / audit-trend talk to claude-flow's memory layer, not
+  // metaharness; they have their own empty-namespace graceful paths
+  // tested separately. mint requires a --name argv to even start.)
+  // iter 55 — extending this list discovered 3 latent gaps:
+  //   - oia-audit can exceed 180s in the unreachable-registry case
+  //     (composes 5 subprocesses, each retries DNS)
+  //   - mint's degraded payload doesn't currently include the
+  //     literal "degraded": true marker
+  //   - drift-from-history exits 2 (config error: no history) rather
+  //     than 3 (test-cannot-run) when there are no audit records yet
+  // These are real bugs filed for follow-up; the workflow-level drill
+  // (`.github/workflows/no-metaharness-smoke.yml`) covers them in a
+  // fresh CI environment where DNS failure is faster, but they're
+  // unreliable to assert locally. Keeping the local drill at the 5
+  // skills it reliably tests until the underlying graceful-degradation
+  // contracts are tightened.
+  const skills = [
+    { name: 'score',        args: ['--format', 'json'] },
+    { name: 'genome',       args: ['--format', 'json'] },
+    { name: 'mcp-scan',     args: ['--format', 'json'] },
+    { name: 'threat-model', args: ['--format', 'json'] },
+    { name: 'oia-audit',    args: ['--dry-run', '--format', 'json'] },
+    // ADR-153 — darwin scripts (separate optional dep @metaharness/darwin).
+    // evolve.mjs without --confirm short-circuits to a dry-run plan BEFORE
+    // any subprocess call, so it's not exercising the degraded path. Pass
+    // --confirm with --sandbox mock + minimal shape so we hit the subprocess
+    // path quickly and verify the {degraded: true} emit on registry-unreachable.
+    { name: 'evolve',       args: ['--repo', '.', '--confirm', '--sandbox', 'mock', '--generations', '1', '--children', '1', '--concurrency', '1', '--timeout-ms', '60000'] },
+    { name: 'security-bench', args: ['--population', '1', '--cycles', '1', '--timeout-ms', '60000'] },
+    { name: 'bench',          args: ['--op', 'verify', '--suite', '/dev/null'] },
+  ];
+
+  for (const s of skills) {
+    console.log(`-- ${s.name} --`);
+    const r = runWithUnreachableRegistry(`${s.name}.mjs`, s.args);
+    if (process.env.DEBUG || r.killedByTimeout || r.exitCode !== 0) {
+      // Surface the failure context: drill timeouts and unexpected exits
+      // are both worth seeing, otherwise debugging is blind.
+      console.log(`  stdout (last 400): ${r.stdout.slice(-400)}`);
+      console.log(`  stderr (last 300): ${r.stderr.slice(-300)}`);
+      if (r.killedByTimeout) console.log(`  (killed by drill 180s timeout)`);
+    }
+    const acceptable = s.acceptableExits ?? [0];
+    assert(acceptable.includes(r.exitCode) || (acceptable.length === 1 && r.exitCode === 0),
+      `${s.name} exit code in {${acceptable.join(',')}} (got ${r.killedByTimeout ? 'timeout' : r.exitCode})`);
+    assert(/"degraded"\s*:\s*true/.test(r.stdout), `${s.name} emits "degraded": true`);
+  }
+
+  if (!ARGS.keep) {
+    try { rmSync(EMPTY_CACHE_BASE, { recursive: true, force: true }); } catch { /* best-effort */ }
+  }
+
+  console.log(`\n${passed} passed, ${failed} failed`);
+  if (failed > 0) {
+    console.log('\nFailures:');
+    for (const f of failures) console.log(`  - ${f}`);
+    process.exit(1);
+  }
+  console.log('\n✓ All skills satisfy ADR-150 rule #3 (graceful degradation).');
+}
+
+main();