@claude-flow/cli 3.10.13 → 3.10.15

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@claude-flow/cli",
-  "version": "3.10.13",
+  "version": "3.10.15",
   "type": "module",
   "description": "Ruflo CLI - Enterprise AI agent orchestration with 60+ specialized agents, swarm coordination, MCP server, self-learning hooks, and vector memory for Claude Code",
   "main": "dist/src/index.js",

package/scripts/benchmark-self-learning.mjs

@@ -0,0 +1,241 @@
+#!/usr/bin/env node
+// benchmark-self-learning.mjs — proof harness for #2245.
+//
+// Replaces the "self-learning reports success but persists nothing" theater
+// with measured deltas: actually runs each entry point N times and prints what
+// each one moved (or didn't). Writes a run JSON to docs/benchmarks/runs/ so
+// future regressions are visible against a committed baseline.
+//
+// Usage:
+//   node scripts/benchmark-self-learning.mjs                  # default N=20 per surface
+//   N=50 BENCH_JSON=1 node scripts/benchmark-self-learning.mjs
+//   BENCH_NO_WRITE=1 node scripts/benchmark-self-learning.mjs
+//
+// Repro:
+//   1. Clone ruvnet/ruflo, npm install, build the CLI:
+//        npm install && (cd v3/@claude-flow/cli && npx tsc -b)
+//   2. Run this script. It prints a "before/after" table per surface.
+//   3. Inspect docs/benchmarks/runs/self-learning-<ts>.json for the persisted
+//      proof — diff against previous runs to catch any future regression.
+
+import { mkdirSync, writeFileSync, mkdtempSync, rmSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+import { performance } from 'node:perf_hooks';
+
+const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
+const CLI_ROOT = resolve(SCRIPT_DIR, '..');
+const REPO_ROOT = resolve(SCRIPT_DIR, '../../../..');
+const RUNS_DIR = join(REPO_ROOT, 'docs', 'benchmarks', 'runs');
+
+const N = Number(process.env.N) || 20;
+
+// Run in an isolated temp dir so we don't pollute the repo's neural store.
+const SCRATCH = mkdtempSync(join(tmpdir(), 'learn-bench-'));
+process.chdir(SCRATCH);
+
+async function main() {
+  const intel = await import(join(CLI_ROOT, 'dist/src/memory/intelligence.js'));
+  const hooks = await import(join(CLI_ROOT, 'dist/src/mcp-tools/hooks-tools.js'));
+  const neural = await import(join(CLI_ROOT, 'dist/src/mcp-tools/neural-tools.js'));
+
+  intel.clearIntelligence();
+
+  // -------------------------------------------------------------------------
+  // §A — recordSignalProcessed
+  // -------------------------------------------------------------------------
+  const A_before = intel.getIntelligenceStats();
+  const tA = performance.now();
+  for (let i = 0; i < N; i++) intel.recordSignalProcessed();
+  intel.flushIntelligenceStats();
+  const dtA = performance.now() - tA;
+  const A_after = intel.getIntelligenceStats();
+  const A_delta = A_after.signalsProcessed - A_before.signalsProcessed;
+
+  // -------------------------------------------------------------------------
+  // §B — hooks_task-completed (trainPatterns:true) → trajectory pipeline
+  // -------------------------------------------------------------------------
+  const taskCompleted = hooks.hooksTools.find((t) => t.name === 'hooks_task-completed');
+  const B_before = intel.getIntelligenceStats();
+  const tB = performance.now();
+  let B_trained = 0;
+  for (let i = 0; i < N; i++) {
+    const r = await taskCompleted.handler({
+      taskId: `bench-${i}`,
+      success: i % 5 !== 4, // 80% success, 20% failure — mixed verdict
+      quality: 0.5 + (i % 5) * 0.1,
+      trainPatterns: true,
+      content: `Benchmark task ${i}: refactor or test or fix`,
+    });
+    if (r.learningPath === 'trajectory-pipeline') B_trained++;
+  }
+  const dtB = performance.now() - tB;
+  const B_after = intel.getIntelligenceStats();
+
+  // -------------------------------------------------------------------------
+  // §C — hooks_task-completed recorded-only (negative control)
+  // -------------------------------------------------------------------------
+  const C_before = intel.getIntelligenceStats();
+  const tC = performance.now();
+  for (let i = 0; i < N; i++) {
+    await taskCompleted.handler({ taskId: `bench-no-train-${i}`, success: true, quality: 0.8 });
+  }
+  const dtC = performance.now() - tC;
+  const C_after = intel.getIntelligenceStats();
+
+  // -------------------------------------------------------------------------
+  // §D — storeNeuralPatterns + neural_patterns list reflects them
+  // -------------------------------------------------------------------------
+  const items = Array.from({ length: N }, (_, i) => ({
+    name: `pattern-${i}`,
+    type: 'bench-pattern',
+    content: `import { thing${i} } from 'module${i}'`,
+  }));
+  const tD = performance.now();
+  const D_store = await neural.storeNeuralPatterns(items);
+  const dtD = performance.now() - tD;
+  const listTool = neural.neuralTools.find((t) => t.name === 'neural_patterns');
+  const D_list = await listTool.handler({ action: 'list' });
+
+  // -------------------------------------------------------------------------
+  // §E — multi-step trajectory pipeline (end-to-end)
+  // -------------------------------------------------------------------------
+  const trajStart = hooks.hooksTools.find((t) => t.name === 'hooks_intelligence_trajectory-start');
+  const trajStep  = hooks.hooksTools.find((t) => t.name === 'hooks_intelligence_trajectory-step');
+  const trajEnd   = hooks.hooksTools.find((t) => t.name === 'hooks_intelligence_trajectory-end');
+  // NOTE: the MCP trajectory tools feed sonaCoordinator (queryable via
+  // hooks_intelligence_stats), NOT globalStats. That's part of the broader
+  // store-fragmentation problem #2245 identifies; we check observable
+  // outcomes (persisted + sonaUpdate) instead of a globalStats delta.
+  const tE = performance.now();
+  let persistedCount = 0;
+  let sonaUpdateCount = 0;
+  if (trajStart && trajStep && trajEnd) {
+    for (let i = 0; i < Math.min(5, N); i++) {
+      const s = await trajStart.handler({ task: `multi-step bench ${i}`, agent: 'bench' });
+      const id = s.trajectoryId;
+      await trajStep.handler({ trajectoryId: id, type: 'observation', content: `obs ${i}` });
+      await trajStep.handler({ trajectoryId: id, type: 'action',      content: `act ${i}` });
+      await trajStep.handler({ trajectoryId: id, type: 'result',      content: `done ${i}` });
+      const endRes = await trajEnd.handler({ trajectoryId: id, success: true });
+      if (endRes.persisted) persistedCount++;
+      if (endRes.learning?.sonaUpdate) sonaUpdateCount++;
+    }
+  }
+  const dtE = performance.now() - tE;
+
+  // -------------------------------------------------------------------------
+  // §F — getUnifiedLearningStats (ADR-075)
+  // -------------------------------------------------------------------------
+  const tF = performance.now();
+  const unified = await intel.getUnifiedLearningStats();
+  const dtF = performance.now() - tF;
+  const unifiedOk = unified.global && unified.sona && unified.memoryBridge && unified.neuralPatterns && unified.consistency;
+
+  // -------------------------------------------------------------------------
+  // Summary
+  // -------------------------------------------------------------------------
+  const summary = {
+    runAt: new Date().toISOString(),
+    benchmark: 'self-learning-2245',
+    n: N,
+    sections: {
+      A_recordSignalProcessed: {
+        calls: N,
+        signalsProcessedDelta: A_delta,
+        passed: A_delta === N,
+        elapsedMs: Number(dtA.toFixed(2)),
+      },
+      B_taskCompleted_trainPatterns: {
+        calls: N,
+        trainedViaPipeline: B_trained,
+        trajectoriesDelta: B_after.trajectoriesRecorded - B_before.trajectoriesRecorded,
+        patternsLearnedDelta: B_after.patternsLearned - B_before.patternsLearned,
+        passed: B_trained === N,
+        elapsedMs: Number(dtB.toFixed(2)),
+        avgLatencyMsPerCall: Number((dtB / N).toFixed(2)),
+      },
+      C_taskCompleted_recordedOnly: {
+        calls: N,
+        trajectoriesDelta: C_after.trajectoriesRecorded - C_before.trajectoriesRecorded,
+        passed: (C_after.trajectoriesRecorded - C_before.trajectoriesRecorded) === 0,
+        note: 'negative control — should NOT touch trajectories',
+        elapsedMs: Number(dtC.toFixed(2)),
+      },
+      D_pretrain_neuralPatterns: {
+        attempted: items.length,
+        stored: D_store.stored,
+        listTotal: D_list.total,
+        passed: D_store.stored === items.length && D_list.total >= items.length,
+        elapsedMs: Number(dtD.toFixed(2)),
+      },
+      E_multiStepTrajectory: {
+        cycles: Math.min(5, N),
+        persistedCount,
+        sonaUpdateCount,
+        passed: persistedCount === Math.min(5, N), // persistence is the must-pass; SONA depends on env
+        note: 'MCP trajectory tools feed sonaCoordinator (see hooks_intelligence_stats), not globalStats — observable outcomes checked here',
+        elapsedMs: Number(dtE.toFixed(2)),
+      },
+      F_unifiedStats: {
+        shape: ['global', 'sona', 'memoryBridge', 'neuralPatterns', 'consistency'],
+        observed: {
+          'global.patternsLearned': unified.global.patternsLearned,
+          'global.trajectoriesRecorded': unified.global.trajectoriesRecorded,
+          'global.signalsProcessed': unified.global.signalsProcessed,
+          'memoryBridge.totalEntries': unified.memoryBridge.totalEntries,
+          'memoryBridge.reachable': unified.memoryBridge.reachable,
+          'neuralPatterns.patternCount': unified.neuralPatterns.patternCount,
+          'sona.available': unified.sona.available,
+          'consistency.notes': unified.consistency.notes.length,
+        },
+        passed: !!unifiedOk,
+        note: 'ADR-075 — one aggregator across the 4 stores. Each sub-view names its source.',
+        elapsedMs: Number(dtF.toFixed(2)),
+      },
+    },
+    finalState: {
+      signalsProcessed: A_after.signalsProcessed,
+      trajectoriesRecorded: intel.getIntelligenceStats().trajectoriesRecorded,
+      patternsLearned: intel.getIntelligenceStats().patternsLearned,
+    },
+    scratch: SCRATCH,
+  };
+
+  const allPassed = Object.values(summary.sections).every((s) => s.passed);
+  summary.passed = allPassed;
+
+  if (process.env.BENCH_JSON) {
+    console.log(JSON.stringify(summary, null, 2));
+  } else {
+    console.log(`# Self-learning benchmark (#2245) — N=${N}`);
+    console.log('');
+    console.log('| Section | Calls | Delta | Passed | Latency (ms) |');
+    console.log('|---|---:|---:|:---:|---:|');
+    console.log(`| A recordSignalProcessed | ${N} | +${summary.sections.A_recordSignalProcessed.signalsProcessedDelta} | ${summary.sections.A_recordSignalProcessed.passed ? '✅' : '❌'} | ${summary.sections.A_recordSignalProcessed.elapsedMs} |`);
+    console.log(`| B task-completed (train) | ${N} | trained=${summary.sections.B_taskCompleted_trainPatterns.trainedViaPipeline}, trajectories+${summary.sections.B_taskCompleted_trainPatterns.trajectoriesDelta} | ${summary.sections.B_taskCompleted_trainPatterns.passed ? '✅' : '❌'} | ${summary.sections.B_taskCompleted_trainPatterns.elapsedMs} (${summary.sections.B_taskCompleted_trainPatterns.avgLatencyMsPerCall}/call) |`);
+    console.log(`| C task-completed (record-only) | ${N} | trajectories+${summary.sections.C_taskCompleted_recordedOnly.trajectoriesDelta} (expected 0) | ${summary.sections.C_taskCompleted_recordedOnly.passed ? '✅' : '❌'} | ${summary.sections.C_taskCompleted_recordedOnly.elapsedMs} |`);
+    console.log(`| D pretrain → neural_patterns | ${items.length} | stored=${summary.sections.D_pretrain_neuralPatterns.stored}, listed=${summary.sections.D_pretrain_neuralPatterns.listTotal} | ${summary.sections.D_pretrain_neuralPatterns.passed ? '✅' : '❌'} | ${summary.sections.D_pretrain_neuralPatterns.elapsedMs} |`);
+    console.log(`| E multi-step trajectory | ${summary.sections.E_multiStepTrajectory.cycles} | persisted=${summary.sections.E_multiStepTrajectory.persistedCount}, sonaUpdate=${summary.sections.E_multiStepTrajectory.sonaUpdateCount} | ${summary.sections.E_multiStepTrajectory.passed ? '✅' : '❌'} | ${summary.sections.E_multiStepTrajectory.elapsedMs} |`);
+    const f = summary.sections.F_unifiedStats;
+    console.log(`| F unified-stats | 4 stores | bridge.reachable=${f.observed['memoryBridge.reachable']}, sona.available=${f.observed['sona.available']}, neural.count=${f.observed['neuralPatterns.patternCount']}, notes=${f.observed['consistency.notes']} | ${f.passed ? '✅' : '❌'} | ${f.elapsedMs} |`);
+    console.log('');
+    console.log(`Final state: signalsProcessed=${summary.finalState.signalsProcessed}, trajectoriesRecorded=${summary.finalState.trajectoriesRecorded}, patternsLearned=${summary.finalState.patternsLearned}`);
+    console.log(`Overall: ${allPassed ? '✅ ALL PASSED' : '❌ FAILED'}`);
+  }
+
+  if (!process.env.BENCH_NO_WRITE) {
+    mkdirSync(RUNS_DIR, { recursive: true });
+    const stamp = summary.runAt.replace(/[:.]/g, '-');
+    writeFileSync(join(RUNS_DIR, `self-learning-${stamp}.json`), JSON.stringify(summary, null, 2));
+    writeFileSync(join(RUNS_DIR, 'self-learning-latest.json'), JSON.stringify(summary, null, 2));
+    if (!process.env.BENCH_JSON) console.log(`\nWrote ${join(RUNS_DIR, `self-learning-${stamp}.json`)}`);
+  }
+
+  try { rmSync(SCRATCH, { recursive: true, force: true }); } catch {}
+
+  if (!allPassed) process.exit(1);
+}
+
+main().catch((err) => { console.error(err); process.exit(1); });