@claude-flow/cli 3.10.17 → 3.10.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@claude-flow/cli",
-  "version": "3.10.17",
+  "version": "3.10.19",
   "type": "module",
   "description": "Ruflo CLI - Enterprise AI agent orchestration with 60+ specialized agents, swarm coordination, MCP server, self-learning hooks, and vector memory for Claude Code",
   "main": "dist/src/index.js",

package/scripts/benchmark-pretrained-retrieval.mjs

@@ -21,19 +21,20 @@ const REPO_ROOT = resolve(SCRIPT_DIR, '../../../..');
 const RUNS_DIR = join(REPO_ROOT, 'docs', 'benchmarks', 'runs');
 
 // Real ruflo-history-shaped queries. Each one targets a concept that should
-// have been seen during pretrain (Opus 4.8 fix, self-learning wiring,
-// codemod engine, etc.).
+// have been seen during pretrain. `expect` is a regex matched against the
+// result's `name` field (commit subject or issue title) to score relevance.
+// A result is "relevant" if its name matches the expect regex (case-insensitive).
 const QUERIES = [
-  'how was the Opus model alias fixed',
-  'self-learning wiring task-completed pretrain',
-  'deterministic codemod engine var-to-const',
-  'MCP server orphan leak parent-death',
-  'unified learning stats aggregator',
-  'structured distillation 4-field schema',
-  'SQL injection migrate.ts table identifier',
-  'recall@k HNSW benchmark harness',
-  'Q-learning encoder keyword block',
-  'security hardening crypto random IDs',
+  { q: 'how was the Opus model alias fixed', expect: /opus|2232|model.*alias|4\.8/i },
+  { q: 'self-learning wiring task-completed pretrain', expect: /self.?learning|task.?completed|pretrain|2245|074/i },
+  { q: 'deterministic codemod engine var-to-const', expect: /codemod|var.?to.?const|143|deterministic/i },
+  { q: 'MCP server orphan leak parent-death', expect: /mcp.*orphan|orphan.*mcp|parent.?death|leak/i },
+  { q: 'unified learning stats aggregator', expect: /unified|stats|aggregator|075/i },
+  { q: 'structured distillation 4-field schema', expect: /distillation|structured|076|4.?field/i },
+  { q: 'SQL injection migrate.ts table identifier', expect: /sql.?injection|migrate|table|identifier/i },
+  { q: 'recall@k HNSW benchmark harness', expect: /recall|hnsw|benchmark|harness/i },
+  { q: 'Q-learning encoder keyword block', expect: /q.?learning|encoder|keyword|2239/i },
+  { q: 'security hardening crypto random IDs', expect: /security|hardening|crypto|random/i },
 ];
 
 const TOP_K = 5;
@@ -51,35 +52,97 @@ async function main() {
     process.exit(2);
   }
 
-  // §2 — run each query through neural_patterns search. The tool's search
-  // action runs cosine similarity against the stored embeddings.
+  // §2 — A/B hybrid vs cosine-only (HYBRID=0 forces pre-3.10.18 behaviour).
+  // Default runs hybrid (cosine + BM25 + MMR per ADR-078).
   const listTool = neural.neuralTools.find((t) => t.name === 'neural_patterns');
+  const mode = process.env.HYBRID === '0' ? 'cosine' : 'hybrid';
+
+  // top-1-uniqueness — fraction of queries whose top-1 result is NOT the
+  // same pattern ID as another query's top-1. Catches the "everyone gets
+  // the same generic top-1" failure mode.
+  const top1Ids = new Map();
+
   const tQuery0 = performance.now();
   const results = [];
-  for (const q of QUERIES) {
-    const r = await listTool.handler({ action: 'search', query: q });
+  for (const { q, expect } of QUERIES) {
+    const r = await listTool.handler({ action: 'search', query: q, mode, limit: TOP_K });
     const matches = (r.patterns || r.results || r.matches || []).slice(0, TOP_K);
+    if (matches.length > 0) {
+      const top1 = matches[0].id;
+      top1Ids.set(top1, (top1Ids.get(top1) ?? 0) + 1);
+    }
+    // ADR-078 relevance: does the top-1/top-3 name match the expect regex?
+    const top1Name = matches[0]?.name ?? '';
+    const top1Relevant = expect.test(top1Name);
+    let top3Relevant = false;
+    let firstRelevantRank = -1;
+    for (let i = 0; i < Math.min(matches.length, 3); i++) {
+      if (expect.test(matches[i]?.name ?? '')) {
+        top3Relevant = true;
+        if (firstRelevantRank === -1) firstRelevantRank = i + 1;
+        break;
+      }
+    }
     results.push({
       query: q,
       matched: matches.length > 0,
+      top1Relevant,
+      top3Relevant,
+      firstRelevantRank,
       topK: matches.map((m) => ({
         id: m.id,
         name: m.name?.slice(0, 100),
         type: m.type,
         score: m.score ?? m.similarity,
+        cosineScore: m.cosineScore,
+        bm25Score: m.bm25Score,
+        mmrScore: m.mmrScore,
       })),
     });
   }
   const queryMs = performance.now() - tQuery0;
 
   const matchedQueries = results.filter((r) => r.matched).length;
+  const top1Hits = results.filter((r) => r.top1Relevant).length;
+  const top3Hits = results.filter((r) => r.top3Relevant).length;
+  const ranks = results.filter((r) => r.firstRelevantRank > 0).map((r) => r.firstRelevantRank);
+  // MRR over the top-3 window (rank-of-first-relevant). Queries with no
+  // relevant result in top-3 contribute 0 to the mean.
+  const mrr3 = QUERIES.length > 0
+    ? Number((ranks.reduce((s, r) => s + 1 / r, 0) / QUERIES.length).toFixed(4))
+    : 0;
+  // top-1 collision: number of distinct top-1 IDs over the query count.
+  const uniqueTop1 = top1Ids.size;
+  const top1Diversity = Number((uniqueTop1 / QUERIES.length).toFixed(4));
+
+  // top-3 redundancy: average fraction of top-3 results that are duplicates
+  // of the same pattern within a single query (should be 0 — but if the same
+  // ID appears twice in top-3 we surface it).
+  let dupCount = 0, top3Slots = 0;
+  for (const r of results) {
+    const ids = r.topK.slice(0, 3).map((m) => m.id);
+    top3Slots += ids.length;
+    const seen = new Set();
+    for (const id of ids) {
+      if (seen.has(id)) dupCount++;
+      seen.add(id);
+    }
+  }
+  const top3DupRate = top3Slots > 0 ? Number((dupCount / top3Slots).toFixed(4)) : 0;
+
   const summary = {
     runAt: new Date().toISOString(),
     benchmark: 'pretrained-retrieval',
+    mode,                              // ADR-078: which retrieval path was used
     storeSize: total,
     queries: QUERIES.length,
     matchedQueries,
     matchRate: Number((matchedQueries / QUERIES.length).toFixed(4)),
+    top1HitRate: Number((top1Hits / QUERIES.length).toFixed(4)),  // ADR-078: relevance, not just any-match
+    top3HitRate: Number((top3Hits / QUERIES.length).toFixed(4)),
+    mrr3,                              // mean reciprocal rank over top-3
+    top1Diversity,                     // 1.0 = every query gets a distinct top-1
+    top3DupRate,                       // 0.0 = no duplicate IDs inside any top-3
     avgQueryLatencyMs: Number((queryMs / QUERIES.length).toFixed(2)),
     totalQueryMs: Number(queryMs.toFixed(2)),
     results,
@@ -90,9 +153,15 @@ async function main() {
     console.log(JSON.stringify(summary, null, 2));
   } else {
     console.log(`# Pretrained-retrieval benchmark — proof of learning`);
+    console.log(`Mode: ${mode}${mode === 'hybrid' ? ' (cosine + BM25 + MMR, ADR-078)' : ' (cosine-only, pre-3.10.18)'}`);
     console.log(`Store size: ${total} patterns`);
     console.log(`Queries: ${QUERIES.length}`);
     console.log(`Match rate: ${(summary.matchRate * 100).toFixed(0)}% (${matchedQueries}/${QUERIES.length})`);
+    console.log(`Top-1 hit rate (RELEVANCE): ${(summary.top1HitRate * 100).toFixed(0)}% (${top1Hits}/${QUERIES.length})`);
+    console.log(`Top-3 hit rate (RELEVANCE): ${(summary.top3HitRate * 100).toFixed(0)}% (${top3Hits}/${QUERIES.length})`);
+    console.log(`MRR@3: ${summary.mrr3}`);
+    console.log(`Top-1 diversity: ${(summary.top1Diversity * 100).toFixed(0)}% (${uniqueTop1} distinct top-1 IDs across ${QUERIES.length} queries)`);
+    console.log(`Top-3 dup rate: ${(summary.top3DupRate * 100).toFixed(0)}%`);
     console.log(`Avg query latency: ${summary.avgQueryLatencyMs} ms`);
     console.log('');
     for (const r of results) {

package/scripts/pretrain-from-github.mjs

@@ -43,29 +43,110 @@ const SOURCE  = process.env.SOURCE || 'all'; // 'all' | 'git' | 'issues'
 // Harvesters
 // ---------------------------------------------------------------------------
 
+// ADR-078 outcome signal — classify each commit by whether it was reverted,
+// hotfix-followed, or stuck. Operates on the harvested-commits window plus a
+// wider lookahead window for revert/hotfix detection.
+//
+// Verdicts emitted:
+//   success — landed cleanly, no later commit reverted or fixed it
+//   reverted — a later commit subject starts with `Revert "<this subject>"`
+//   hotfixed — a later commit (within HOTFIX_WINDOW_COMMITS) shares >=50%
+//              of the same touched files AND subject contains fix|hotfix|patch
+//
+// The "later" direction is git-log order (newest first → we look at older
+// indices in the lookahead, which are NEWER commits chronologically).
+const HOTFIX_WINDOW_COMMITS = Number(process.env.HOTFIX_WINDOW_COMMITS) || 20;
+const HOTFIX_FILE_OVERLAP = Number(process.env.HOTFIX_FILE_OVERLAP) || 0.5;
+const HOTFIX_KEYWORDS = /\b(fix|hotfix|patch|revert|bugfix|fixup)\b/i;
+
 function harvestCommits(n) {
   if (SOURCE === 'issues') return [];
   const fmt = '%H%x00%s%x00%b%x01';
+  // Pull a wider window so we have lookahead for revert/hotfix detection
+  // without changing the trained set size n.
+  const lookahead = HOTFIX_WINDOW_COMMITS;
   const raw = execSync(
-    `git log --pretty=format:'${fmt}' -n ${n} 2>/dev/null`,
+    `git log --pretty=format:'${fmt}' -n ${n + lookahead} 2>/dev/null`,
     { cwd: REPO_ROOT, encoding: 'utf-8', maxBuffer: 64 * 1024 * 1024 },
   );
-  const entries = [];
+
+  // Parse all blocks (window + lookahead).
+  const all = [];
   for (const block of raw.split('\x01')) {
     if (!block.trim()) continue;
     const [sha, subject, body] = block.split('\x00');
     if (!sha || !subject) continue;
+    all.push({ sha, subject: subject.trim(), body: (body || '').trim() });
+  }
+
+  // For each commit in the trained slice, get its touched files.
+  const filesCache = new Map();
+  const filesOf = (sha) => {
+    if (filesCache.has(sha)) return filesCache.get(sha);
+    try {
+      const out = execSync(
+        `git show --pretty=format: --name-only ${sha} 2>/dev/null`,
+        { cwd: REPO_ROOT, encoding: 'utf-8', maxBuffer: 8 * 1024 * 1024 },
+      ).trim().split('\n').filter(Boolean);
+      filesCache.set(sha, new Set(out));
+      return filesCache.get(sha);
+    } catch {
+      filesCache.set(sha, new Set());
+      return filesCache.get(sha);
+    }
+  };
+
+  // Build a revert-target map: any commit whose subject starts with
+  // `Revert "<X>"` flags X as reverted.
+  const revertedSubjects = new Set();
+  for (const c of all) {
+    const m = c.subject.match(/^Revert\s+"(.+?)"/);
+    if (m) revertedSubjects.add(m[1].trim());
+  }
+
+  const trained = all.slice(0, n);
+  const entries = [];
+  for (let i = 0; i < trained.length; i++) {
+    const c = trained[i];
+    const myFiles = filesOf(c.sha);
+
+    let verdict = 'success';
+    let outcomeNote = null;
+
+    // Reverted? Check if any later (lower-index = newer) commit reverted us.
+    if (revertedSubjects.has(c.subject)) {
+      verdict = 'reverted';
+      outcomeNote = 'subject reverted by a later commit';
+    } else {
+      // Hotfixed? A later commit within HOTFIX_WINDOW_COMMITS shares files
+      // AND has fix/hotfix in its subject.
+      const start = Math.max(0, i - HOTFIX_WINDOW_COMMITS);
+      for (let j = start; j < i; j++) {
+        const later = trained[j];
+        if (!HOTFIX_KEYWORDS.test(later.subject)) continue;
+        const laterFiles = filesOf(later.sha);
+        if (laterFiles.size === 0 || myFiles.size === 0) continue;
+        let overlap = 0;
+        for (const f of laterFiles) if (myFiles.has(f)) overlap++;
+        // Use min() so a small targeted fix on a big change still triggers
+        // (semantic: "≥half of the smaller commit's files overlap").
+        const overlapFrac = overlap / Math.min(laterFiles.size, myFiles.size);
+        if (overlapFrac >= HOTFIX_FILE_OVERLAP) {
+          verdict = 'hotfixed';
+          outcomeNote = `${(overlapFrac * 100).toFixed(0)}% file overlap with later fix ${later.sha.slice(0, 8)}`;
+          break;
+        }
+      }
+    }
+
     entries.push({
       source: 'commit',
-      id: `commit-${sha.slice(0, 12)}`,
-      subject: subject.trim(),
-      body: (body || '').trim(),
-      // The "verdict" of a commit isn't directly known. Treat all commits as
-      // success — we're learning from intent, not outcome. Failures are
-      // captured separately via the issue stream (closed issues are
-      // success; open issues are partial).
-      verdict: 'success',
-      content: `${subject.trim()}\n\n${(body || '').trim()}`.slice(0, 8192),
+      id: `commit-${c.sha.slice(0, 12)}`,
+      subject: c.subject,
+      body: c.body,
+      verdict,
+      outcomeNote,
+      content: `${c.subject}\n\n${c.body}`.slice(0, 8192),
     });
   }
   return entries;
@@ -126,7 +207,19 @@ async function main() {
     console.log(`Harvested: ${commits.length} commits + ${issues.length} issues = ${items.length} trajectories (${harvestMs.toFixed(0)} ms)`);
   }
 
-  // §3 — feed each item through the trajectory pipeline.
+  // §3 — feed each item through the trajectory pipeline. The harvester
+  // emits one of: success | partial | reverted | hotfixed. We map this to
+  // the trajectory pipeline's binary verdict {success, partial} since the
+  // pipeline doesn't accept arbitrary strings — but we preserve the original
+  // outcome in metadata + the summary's verdictMix so the signal isn't lost.
+  const verdictToPipeline = (v) => {
+    if (v === 'success') return 'success';
+    if (v === 'partial') return 'partial';
+    if (v === 'reverted') return 'partial';   // a revert is the strongest "this was wrong" signal we have
+    if (v === 'hotfixed') return 'partial';   // a same-files fix-followup is "needs adjustment"
+    return 'success';
+  };
+  const verdictMix = { success: 0, partial: 0, reverted: 0, hotfixed: 0 };
   const tFeed0 = performance.now();
   let trained = 0;
   let failed = 0;
@@ -134,6 +227,7 @@ async function main() {
   for (const item of items) {
     try {
       const distilled = distillAndSerialise(item.content);
+      verdictMix[item.verdict] = (verdictMix[item.verdict] ?? 0) + 1;
       await intel.recordTrajectory(
         [{
           type: 'result',
@@ -142,10 +236,12 @@ async function main() {
             source: item.source,
             id: item.id,
             subject: item.subject.slice(0, 200),
+            outcomeVerdict: item.verdict,
+            outcomeNote: item.outcomeNote ?? null,
           },
           timestamp: Date.now(),
         }],
-        item.verdict,
+        verdictToPipeline(item.verdict),
       );
       trained++;
     } catch (err) {
@@ -201,6 +297,7 @@ async function main() {
       avgLatencyMs: items.length > 0 ? Number((feedMs / items.length).toFixed(2)) : 0,
       totalMs: Number(feedMs.toFixed(2)),
       sampleFailures: failures,
+      verdictMix,   // ADR-078 outcome signal — counts per harvested verdict
     },
     seedNeuralStore: {
       stored: seedResult.stored,
@@ -231,6 +328,7 @@ async function main() {
     console.log(`Failed: ${failed}`);
     console.log(`Avg latency per trajectory: ${summary.feed.avgLatencyMs} ms`);
     console.log(`Neural store seeded: ${seedResult.stored}/${seedResult.total}`);
+    console.log(`Verdict mix: success=${verdictMix.success} partial=${verdictMix.partial} reverted=${verdictMix.reverted} hotfixed=${verdictMix.hotfixed}`);
     console.log(`Overall: ${summary.passed ? '✅ PASSED' : '⚠️  partial'}`);
     if (unified1.consistency.notes.length > 0) {
       console.log(`\nConsistency notes:`);