@claude-flow/cli 3.10.24 → 3.10.26

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@claude-flow/cli",
-  "version": "3.10.24",
+  "version": "3.10.26",
   "type": "module",
   "description": "Ruflo CLI - Enterprise AI agent orchestration with 60+ specialized agents, swarm coordination, MCP server, self-learning hooks, and vector memory for Claude Code",
   "main": "dist/src/index.js",

package/scripts/beir-bootstrap-significance.mjs

@@ -0,0 +1,162 @@
+#!/usr/bin/env node
+// beir-bootstrap-significance.mjs — paired bootstrap significance test for
+// BEIR retrieval results (ADR-086).
+//
+// Given a run JSON with per-query nDCG@10 and a baseline mean nDCG@10,
+// estimates whether our result is significantly above the baseline.
+//
+// Method: paired bootstrap — resample N=10000 with replacement from our
+// per-query nDCG scores, compute mean each time, take 2.5%-97.5% percentile
+// for 95% CI. Compare baseline to our CI: if baseline < CI lower bound,
+// the difference is significant at p < 0.05.
+//
+// For a paired test against ANOTHER per-query run (e.g. our pure-BM25 vs
+// our BGE-base), resample paired DIFFERENCES instead. Pass --paired
+// path/to/baseline-run.json.
+//
+// Usage:
+//   node scripts/beir-bootstrap-significance.mjs <run-json-path>
+//   node scripts/beir-bootstrap-significance.mjs <ours> --paired <baseline>
+//
+// Example:
+//   node scripts/beir-bootstrap-significance.mjs \
+//     docs/benchmarks/runs/beir-nfcorpus-bge-latest.json
+
+import { readFileSync } from 'node:fs';
+import { performance } from 'node:perf_hooks';
+
+const args = process.argv.slice(2);
+if (args.length === 0) {
+  console.error('Usage: node beir-bootstrap-significance.mjs <run-json> [--paired <baseline-run-json>]');
+  process.exit(1);
+}
+const RUN_PATH = args[0];
+const PAIRED_IDX = args.indexOf('--paired');
+const PAIRED_PATH = PAIRED_IDX > 0 ? args[PAIRED_IDX + 1] : null;
+const ITERATIONS = Number(process.env.BOOTSTRAP_ITERATIONS) || 10000;
+
+// Deterministic PRNG (Mulberry32) — so re-runs are reproducible.
+const SEED = Number(process.env.BOOTSTRAP_SEED) || 42;
+function mulberry32(a) {
+  return function () {
+    a |= 0; a = (a + 0x6D2B79F5) | 0;
+    let t = a;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+function mean(arr) { let s = 0; for (const v of arr) s += v; return s / arr.length; }
+
+function percentile(sortedArr, p) {
+  const idx = Math.max(0, Math.min(sortedArr.length - 1, Math.floor(p * sortedArr.length)));
+  return sortedArr[idx];
+}
+
+function pointEstimate(scores) {
+  return {
+    mean: mean(scores),
+    n: scores.length,
+  };
+}
+
+function bootstrapCI(scores, iterations, rng, percentiles = [0.025, 0.975]) {
+  const means = new Float64Array(iterations);
+  const n = scores.length;
+  for (let i = 0; i < iterations; i++) {
+    let s = 0;
+    for (let j = 0; j < n; j++) s += scores[Math.floor(rng() * n)];
+    means[i] = s / n;
+  }
+  const sorted = Array.from(means).sort((a, b) => a - b);
+  return {
+    point: mean(scores),
+    lower: percentile(sorted, percentiles[0]),
+    upper: percentile(sorted, percentiles[1]),
+    p_value_vs_baseline_zero: null,
+  };
+}
+
+// Paired bootstrap: compare our per-query scores to a baseline's per-query
+// scores. Resample (idx) and average paired differences. The 95% CI on the
+// mean difference tells us if it's significantly non-zero.
+function pairedBootstrap(oursByQid, baselineByQid, iterations, rng) {
+  const sharedIds = [...oursByQid.keys()].filter((id) => baselineByQid.has(id));
+  const diffs = sharedIds.map((id) => oursByQid.get(id) - baselineByQid.get(id));
+  const n = diffs.length;
+  const means = new Float64Array(iterations);
+  for (let i = 0; i < iterations; i++) {
+    let s = 0;
+    for (let j = 0; j < n; j++) s += diffs[Math.floor(rng() * n)];
+    means[i] = s / n;
+  }
+  const sorted = Array.from(means).sort((a, b) => a - b);
+  // One-sided p ≈ fraction of resamples where mean diff ≤ 0
+  let negCount = 0;
+  for (const m of means) if (m <= 0) negCount++;
+  return {
+    n_paired: n,
+    point_diff: mean(diffs),
+    lower_95: percentile(sorted, 0.025),
+    upper_95: percentile(sorted, 0.975),
+    p_one_sided: negCount / iterations,
+  };
+}
+
+function loadRun(path) {
+  const data = JSON.parse(readFileSync(path, 'utf-8'));
+  if (!data.perQuery || !Array.isArray(data.perQuery)) {
+    console.error(`Run JSON at ${path} has no perQuery array — re-run the bench with the updated harness.`);
+    process.exit(2);
+  }
+  return data;
+}
+
+const ours = loadRun(RUN_PATH);
+console.log(`# BEIR significance test (ADR-086)`);
+console.log(`Run: ${RUN_PATH}`);
+console.log(`Dataset: ${ours.dataset ?? '?'}  Model: ${ours.model ?? '?'}  Queries: ${ours.perQuery.length}`);
+console.log(`Bootstrap: ${ITERATIONS} iterations, seed=${SEED}`);
+
+const rng = mulberry32(SEED);
+const ourScores = ours.perQuery.map((q) => q.ndcg10);
+
+const t0 = performance.now();
+const oursCI = bootstrapCI(ourScores, ITERATIONS, rng);
+console.log(`\n=== Our nDCG@10 (1-sample bootstrap CI) ===`);
+console.log(`  point:    ${oursCI.point.toFixed(4)}`);
+console.log(`  95% CI:   [${oursCI.lower.toFixed(4)}, ${oursCI.upper.toFixed(4)}]`);
+
+if (ours.baselines) {
+  console.log(`\n=== vs each published baseline (CI overlap) ===`);
+  for (const [name, score] of Object.entries(ours.baselines)) {
+    const ciDiff = oursCI.point - score;
+    const lowerDiff = oursCI.lower - score;
+    const upperDiff = oursCI.upper - score;
+    const direction = ciDiff > 0 ? '↑ above' : '↓ below';
+    const sig = (lowerDiff > 0 && upperDiff > 0) ? 'p<0.05 (95% CI excludes baseline)'
+              : (lowerDiff < 0 && upperDiff < 0) ? 'p<0.05 (95% CI below baseline)'
+              : 'n.s. (95% CI overlaps baseline)';
+    console.log(`  ${score.toFixed(3)}  ${name.padEnd(28)}  Δ=${ciDiff >= 0 ? '+' : ''}${ciDiff.toFixed(4)}  ${direction}  [${sig}]`);
+  }
+}
+
+if (PAIRED_PATH) {
+  const baseline = loadRun(PAIRED_PATH);
+  console.log(`\n=== Paired bootstrap vs ${PAIRED_PATH} ===`);
+  const oursByQid = new Map(ours.perQuery.map((q) => [q.qid, q.ndcg10]));
+  const baseByQid = new Map(baseline.perQuery.map((q) => [q.qid, q.ndcg10]));
+  const rng2 = mulberry32(SEED + 1);
+  const paired = pairedBootstrap(oursByQid, baseByQid, ITERATIONS, rng2);
+  console.log(`  paired queries: ${paired.n_paired}`);
+  console.log(`  Δ (ours - baseline): ${paired.point_diff >= 0 ? '+' : ''}${paired.point_diff.toFixed(4)}`);
+  console.log(`  95% CI: [${paired.lower_95.toFixed(4)}, ${paired.upper_95.toFixed(4)}]`);
+  console.log(`  one-sided p (Δ ≤ 0): ${paired.p_one_sided.toFixed(4)}`);
+  const sig = paired.lower_95 > 0 ? 'SIGNIFICANT improvement (p<0.025 one-sided)' :
+              paired.upper_95 < 0 ? 'SIGNIFICANT regression (p<0.025 one-sided)' :
+              'NOT SIGNIFICANT — CI overlaps zero';
+  console.log(`  verdict: ${sig}`);
+}
+
+console.log(`\nBootstrap took ${((performance.now() - t0) / 1000).toFixed(1)}s.`);

package/scripts/run-beir-bge.mjs

@@ -0,0 +1,294 @@
+#!/usr/bin/env node
+// run-beir-bge.mjs — NFCorpus retrieval using real BGE-base-en-v1.5
+// embeddings (bypasses neural-tools' broken agentic-flow path that hash-fell-
+// back on darwin-arm64 without sharp/libvips).
+//
+// Stores doc embeddings in a single .f32 binary file (3633 * 768 * 4 = ~11MB)
+// for fast bench iteration without re-embedding.
+//
+// Usage:
+//   cd /tmp/beir-nfcorpus
+//   node /path/to/scripts/run-beir-bge.mjs                          # full ingest + bench
+//   SKIP_INGEST=1 node /path/to/scripts/run-beir-bge.mjs            # reuse cached embeds
+//   BGE_MODEL=Xenova/bge-small-en-v1.5 node ...                     # faster, lower-quality
+//   BGE_MODEL=Xenova/bge-large-en-v1.5 node ...                     # slower, higher-quality
+//
+// ADR-086.
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join, resolve } from 'node:path';
+import { performance } from 'node:perf_hooks';
+
+const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
+const CLI_ROOT = resolve(SCRIPT_DIR, '..');
+const RUFLO_ROOT = resolve(SCRIPT_DIR, '../../../..');
+const RUNS_DIR = join(RUFLO_ROOT, 'docs', 'benchmarks', 'runs');
+
+const DATA_DIR = process.env.BEIR_DATA_DIR || '/tmp/beir-nfcorpus/nfcorpus';
+const CACHE_DIR = join('/tmp/beir-nfcorpus', 'bge-cache');
+const BGE_MODEL = process.env.BGE_MODEL || 'Xenova/bge-base-en-v1.5';
+const SKIP_INGEST = process.env.SKIP_INGEST === '1';
+const MAX_QUERIES = Number(process.env.MAX_QUERIES) || 0;
+
+// Published BEIR baselines per dataset (nDCG@10).
+// Sources: Thakur et al. 2021 (BEIR paper), BGE paper (BAAI 2024),
+// SPLADE++ paper, papers-with-code BEIR leaderboards.
+const BASELINES_BY_DATASET = {
+  nfcorpus: {
+    'BM25 (Lucene)':        0.325,
+    'DocT5query':           0.328,
+    'TAS-B':                0.319,
+    'GenQ':                 0.319,
+    'ColBERT':              0.305,
+    'Contriever':           0.328,
+    'GTR-XL':               0.343,
+    'SPLADE++':             0.347,
+    'BGE-large-v1.5 (pub)': 0.380,
+    'SBERT msmarco':        0.272,
+  },
+  scifact: {
+    'BM25 (Lucene)':        0.679,
+    'DocT5query':           0.675,
+    'TAS-B':                0.643,
+    'GenQ':                 0.644,
+    'ColBERT':              0.671,
+    'Contriever':           0.677,
+    'GTR-XL':               0.662,
+    'SPLADE++':             0.704,
+    'BGE-large-v1.5 (pub)': 0.722,
+    'SBERT msmarco':        0.555,
+  },
+};
+
+// Auto-detect dataset from DATA_DIR path; default to nfcorpus baselines.
+function detectDataset(path) {
+  const p = path.toLowerCase();
+  for (const ds of Object.keys(BASELINES_BY_DATASET)) {
+    if (p.includes(ds)) return ds;
+  }
+  return 'nfcorpus';
+}
+
+// ---------------------------------------------------------------------------
+// nDCG (graded)
+// ---------------------------------------------------------------------------
+
+function dcg(rels, k) {
+  let s = 0;
+  for (let i = 0; i < Math.min(rels.length, k); i++) {
+    s += (Math.pow(2, rels[i]) - 1) / Math.log2(i + 2);
+  }
+  return s;
+}
+function ndcg(retrieved, qrels, k) {
+  const rels = retrieved.slice(0, k).map((id) => qrels.get(id) ?? 0);
+  const ideal = [...qrels.values()].sort((a, b) => b - a).slice(0, k);
+  const idcg = dcg(ideal, k);
+  return idcg > 0 ? dcg(rels, k) / idcg : 0;
+}
+function mrr(retrieved, qrels, k) {
+  for (let i = 0; i < Math.min(retrieved.length, k); i++) {
+    if ((qrels.get(retrieved[i]) ?? 0) > 0) return 1 / (i + 1);
+  }
+  return 0;
+}
+function recall(retrieved, qrels, k) {
+  const tot = [...qrels.values()].filter((v) => v > 0).length;
+  if (tot === 0) return 0;
+  let h = 0;
+  for (let i = 0; i < Math.min(retrieved.length, k); i++) {
+    if ((qrels.get(retrieved[i]) ?? 0) > 0) h++;
+  }
+  return h / tot;
+}
+
+function cosine(a, b) {
+  let s = 0;
+  for (let i = 0; i < a.length; i++) s += a[i] * b[i];
+  return s; // already L2-normalised in BGE
+}
+
+// ---------------------------------------------------------------------------
+// Loaders
+// ---------------------------------------------------------------------------
+
+function loadJsonl(path) {
+  return readFileSync(path, 'utf-8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
+}
+function loadQrels(path) {
+  const qrels = new Map();
+  const lines = readFileSync(path, 'utf-8').split('\n');
+  for (let i = 1; i < lines.length; i++) {
+    if (!lines[i].trim()) continue;
+    const [qid, did, score] = lines[i].split('\t');
+    if (!qrels.has(qid)) qrels.set(qid, new Map());
+    qrels.get(qid).set(did, Number(score));
+  }
+  return qrels;
+}
+
+// ---------------------------------------------------------------------------
+// Embedding cache I/O — single .f32 file + .ids file
+// ---------------------------------------------------------------------------
+
+function cachePath(suffix) {
+  const safe = BGE_MODEL.replace(/\//g, '_');
+  return join(CACHE_DIR, `${safe}.${suffix}`);
+}
+
+function saveEmbeddings(ids, embeddings, dim) {
+  mkdirSync(CACHE_DIR, { recursive: true });
+  // ids: one per line
+  writeFileSync(cachePath('ids'), ids.join('\n') + '\n');
+  // embeddings: concatenated Float32Array
+  const buf = new Float32Array(ids.length * dim);
+  for (let i = 0; i < ids.length; i++) buf.set(embeddings[i], i * dim);
+  writeFileSync(cachePath('f32'), Buffer.from(buf.buffer));
+}
+
+function loadEmbeddings(dim) {
+  if (!existsSync(cachePath('ids')) || !existsSync(cachePath('f32'))) return null;
+  const ids = readFileSync(cachePath('ids'), 'utf-8').split('\n').filter(Boolean);
+  const raw = readFileSync(cachePath('f32'));
+  const buf = new Float32Array(raw.buffer, raw.byteOffset, raw.length / 4);
+  const embeds = [];
+  for (let i = 0; i < ids.length; i++) embeds.push(buf.slice(i * dim, (i + 1) * dim));
+  return { ids, embeds };
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+async function main() {
+  const dataset = detectDataset(DATA_DIR);
+  const BASELINES_NDCG10 = BASELINES_BY_DATASET[dataset];
+  console.log(`# BEIR ${dataset} — BGE embedder (ADR-085/086)`);
+  console.log(`Model: ${BGE_MODEL}`);
+  console.log(`Data:  ${DATA_DIR}`);
+
+  const corpus = loadJsonl(join(DATA_DIR, 'corpus.jsonl'));
+  const queries = loadJsonl(join(DATA_DIR, 'queries.jsonl'));
+  const qrels = loadQrels(join(DATA_DIR, 'qrels/test.tsv'));
+  console.log(`Corpus: ${corpus.length} docs · Queries: ${queries.length} · Test qrels: ${qrels.size}`);
+
+  const bge = await import(join(CLI_ROOT, 'dist/src/memory/bge-embedder.js'));
+  const emb = await bge.getBgeEmbedder(BGE_MODEL);
+  if (!emb) {
+    console.error('BGE failed to load:', bge.getBgeStatus());
+    process.exit(1);
+  }
+  const dim = emb.dim();
+  console.log(`BGE loaded (dim=${dim})`);
+
+  // Ingest or load cached doc embeddings.
+  let docIds, docEmbeds;
+  const cached = SKIP_INGEST ? loadEmbeddings(dim) : null;
+  if (cached) {
+    docIds = cached.ids;
+    docEmbeds = cached.embeds;
+    console.log(`Loaded ${docIds.length} cached doc embeddings from ${cachePath('f32')}`);
+  } else {
+    console.log(`\nEmbedding ${corpus.length} docs (this is the slow step, ~20 min on M-series CPU)...`);
+    docIds = corpus.map((d) => d._id);
+    docEmbeds = new Array(corpus.length);
+    const tIngest = performance.now();
+    for (let i = 0; i < corpus.length; i++) {
+      const d = corpus[i];
+      const text = `${d.title || ''}\n${d.text || ''}`.slice(0, 4096);
+      docEmbeds[i] = await emb.embed(text);
+      if ((i + 1) % 100 === 0) {
+        const elapsed = (performance.now() - tIngest) / 1000;
+        const rate = (i + 1) / elapsed;
+        const eta = (corpus.length - i - 1) / rate;
+        console.log(`  ${i + 1}/${corpus.length}  ${rate.toFixed(1)} docs/s  ETA ${eta.toFixed(0)}s`);
+      }
+    }
+    console.log(`Embedded ${corpus.length} docs in ${((performance.now() - tIngest) / 1000).toFixed(0)}s`);
+    saveEmbeddings(docIds, docEmbeds, dim);
+    console.log(`Cached to ${cachePath('f32')}`);
+  }
+
+  // Run retrieval per query.
+  const queriesById = new Map(queries.map((q) => [q._id, q.text]));
+  const evalQueryIds = MAX_QUERIES
+    ? [...qrels.keys()].slice(0, MAX_QUERIES)
+    : [...qrels.keys()];
+
+  console.log(`\nRunning ${evalQueryIds.length} queries...`);
+  let nSum = 0, mSum = 0, r10Sum = 0, r100Sum = 0, n = 0;
+  // ADR-086 — save per-query metrics for paired bootstrap significance testing.
+  const perQuery = [];
+  const tQ = performance.now();
+  for (const qid of evalQueryIds) {
+    const qtext = queriesById.get(qid);
+    if (!qtext) continue;
+    // BGE v1.5 non-icl works without the query prefix.
+    const qEmb = await emb.embed(qtext);
+    const scores = new Array(docEmbeds.length);
+    for (let i = 0; i < docEmbeds.length; i++) scores[i] = { id: docIds[i], score: cosine(qEmb, docEmbeds[i]) };
+    scores.sort((a, b) => b.score - a.score);
+    const top100 = scores.slice(0, 100).map((s) => s.id);
+
+    const qmap = qrels.get(qid);
+    const qNdcg = ndcg(top100, qmap, 10);
+    const qMrr = mrr(top100, qmap, 10);
+    const qR10 = recall(top100, qmap, 10);
+    const qR100 = recall(top100, qmap, 100);
+    nSum += qNdcg; mSum += qMrr; r10Sum += qR10; r100Sum += qR100; n++;
+    perQuery.push({ qid, ndcg10: qNdcg, mrr10: qMrr, recall10: qR10, recall100: qR100 });
+    if (n % 50 === 0) {
+      const elapsed = (performance.now() - tQ) / 1000;
+      console.log(`  ${n}/${evalQueryIds.length} in ${elapsed.toFixed(0)}s  running nDCG@10=${(nSum / n).toFixed(4)}`);
+    }
+  }
+  const queryMs = performance.now() - tQ;
+
+  const ndcg10 = nSum / n;
+  const mrr10 = mSum / n;
+  const recall10 = r10Sum / n;
+  const recall100 = r100Sum / n;
+
+  console.log(`\n=== Results (N=${n}, BGE-base-en-v1.5) ===`);
+  console.log(`  nDCG@10:    ${ndcg10.toFixed(4)}`);
+  console.log(`  MRR@10:     ${mrr10.toFixed(4)}`);
+  console.log(`  Recall@10:  ${recall10.toFixed(4)}`);
+  console.log(`  Recall@100: ${recall100.toFixed(4)}`);
+  console.log(`  Avg query latency: ${(queryMs / n).toFixed(0)}ms`);
+
+  console.log(`\n=== vs published NFCorpus baselines (nDCG@10) ===`);
+  const ourLabel = `ruflo + ${BGE_MODEL.replace('Xenova/', '')}`;
+  const ranking = [
+    ...Object.entries(BASELINES_NDCG10).map(([name, score]) => ({ name, score, ours: false })),
+    { name: ourLabel, score: ndcg10, ours: true },
+  ].sort((a, b) => b.score - a.score);
+  for (const r of ranking) {
+    console.log(`  ${r.score.toFixed(3)}  ${r.name}${r.ours ? ' ← us' : ''}`);
+  }
+  const ourRank = ranking.findIndex((r) => r.ours) + 1;
+  console.log(`\n  Our rank: ${ourRank} / ${ranking.length}`);
+
+  const summary = {
+    runAt: new Date().toISOString(),
+    benchmark: `beir-${dataset}-bge`,
+    dataset,
+    model: BGE_MODEL,
+    queries: n,
+    corpusSize: corpus.length,
+    metrics: { ndcg10, mrr10, recall10, recall100, avgQueryLatencyMs: queryMs / n },
+    perQuery, // ADR-086: per-query metrics for paired bootstrap significance testing
+    baselines: BASELINES_NDCG10,
+    ourRank,
+    leaderboardLength: ranking.length,
+  };
+  mkdirSync(RUNS_DIR, { recursive: true });
+  const stamp = summary.runAt.replace(/[:.]/g, '-');
+  const safe = BGE_MODEL.replace(/\//g, '_');
+  writeFileSync(join(RUNS_DIR, `beir-${dataset}-bge-${safe}-${stamp}.json`), JSON.stringify(summary, null, 2));
+  writeFileSync(join(RUNS_DIR, `beir-${dataset}-bge-latest.json`), JSON.stringify(summary, null, 2));
+  console.log(`\nWrote ${join(RUNS_DIR, `beir-${dataset}-bge-${safe}-${stamp}.json`)}`);
+  process.exit(0);
+}
+
+main().catch((err) => { console.error(err); process.exit(1); });