@claude-flow/cli 3.10.25 → 3.10.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@claude-flow/cli",
-  "version": "3.10.25",
+  "version": "3.10.27",
   "type": "module",
   "description": "Ruflo CLI - Enterprise AI agent orchestration with 60+ specialized agents, swarm coordination, MCP server, self-learning hooks, and vector memory for Claude Code",
   "main": "dist/src/index.js",

package/scripts/beir-bootstrap-significance.mjs

@@ -0,0 +1,162 @@
+#!/usr/bin/env node
+// beir-bootstrap-significance.mjs — paired bootstrap significance test for
+// BEIR retrieval results (ADR-086).
+//
+// Given a run JSON with per-query nDCG@10 and a baseline mean nDCG@10,
+// estimates whether our result is significantly above the baseline.
+//
+// Method: paired bootstrap — resample N=10000 with replacement from our
+// per-query nDCG scores, compute mean each time, take 2.5%-97.5% percentile
+// for 95% CI. Compare baseline to our CI: if baseline < CI lower bound,
+// the difference is significant at p < 0.05.
+//
+// For a paired test against ANOTHER per-query run (e.g. our pure-BM25 vs
+// our BGE-base), resample paired DIFFERENCES instead. Pass --paired
+// path/to/baseline-run.json.
+//
+// Usage:
+//   node scripts/beir-bootstrap-significance.mjs <run-json-path>
+//   node scripts/beir-bootstrap-significance.mjs <ours> --paired <baseline>
+//
+// Example:
+//   node scripts/beir-bootstrap-significance.mjs \
+//     docs/benchmarks/runs/beir-nfcorpus-bge-latest.json
+
+import { readFileSync } from 'node:fs';
+import { performance } from 'node:perf_hooks';
+
+const args = process.argv.slice(2);
+if (args.length === 0) {
+  console.error('Usage: node beir-bootstrap-significance.mjs <run-json> [--paired <baseline-run-json>]');
+  process.exit(1);
+}
+const RUN_PATH = args[0];
+const PAIRED_IDX = args.indexOf('--paired');
+const PAIRED_PATH = PAIRED_IDX > 0 ? args[PAIRED_IDX + 1] : null;
+const ITERATIONS = Number(process.env.BOOTSTRAP_ITERATIONS) || 10000;
+
+// Deterministic PRNG (Mulberry32) — so re-runs are reproducible.
+const SEED = Number(process.env.BOOTSTRAP_SEED) || 42;
+function mulberry32(a) {
+  return function () {
+    a |= 0; a = (a + 0x6D2B79F5) | 0;
+    let t = a;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+function mean(arr) { let s = 0; for (const v of arr) s += v; return s / arr.length; }
+
+function percentile(sortedArr, p) {
+  const idx = Math.max(0, Math.min(sortedArr.length - 1, Math.floor(p * sortedArr.length)));
+  return sortedArr[idx];
+}
+
+function pointEstimate(scores) {
+  return {
+    mean: mean(scores),
+    n: scores.length,
+  };
+}
+
+function bootstrapCI(scores, iterations, rng, percentiles = [0.025, 0.975]) {
+  const means = new Float64Array(iterations);
+  const n = scores.length;
+  for (let i = 0; i < iterations; i++) {
+    let s = 0;
+    for (let j = 0; j < n; j++) s += scores[Math.floor(rng() * n)];
+    means[i] = s / n;
+  }
+  const sorted = Array.from(means).sort((a, b) => a - b);
+  return {
+    point: mean(scores),
+    lower: percentile(sorted, percentiles[0]),
+    upper: percentile(sorted, percentiles[1]),
+    p_value_vs_baseline_zero: null,
+  };
+}
+
+// Paired bootstrap: compare our per-query scores to a baseline's per-query
+// scores. Resample (idx) and average paired differences. The 95% CI on the
+// mean difference tells us if it's significantly non-zero.
+function pairedBootstrap(oursByQid, baselineByQid, iterations, rng) {
+  const sharedIds = [...oursByQid.keys()].filter((id) => baselineByQid.has(id));
+  const diffs = sharedIds.map((id) => oursByQid.get(id) - baselineByQid.get(id));
+  const n = diffs.length;
+  const means = new Float64Array(iterations);
+  for (let i = 0; i < iterations; i++) {
+    let s = 0;
+    for (let j = 0; j < n; j++) s += diffs[Math.floor(rng() * n)];
+    means[i] = s / n;
+  }
+  const sorted = Array.from(means).sort((a, b) => a - b);
+  // One-sided p ≈ fraction of resamples where mean diff ≤ 0
+  let negCount = 0;
+  for (const m of means) if (m <= 0) negCount++;
+  return {
+    n_paired: n,
+    point_diff: mean(diffs),
+    lower_95: percentile(sorted, 0.025),
+    upper_95: percentile(sorted, 0.975),
+    p_one_sided: negCount / iterations,
+  };
+}
+
+function loadRun(path) {
+  const data = JSON.parse(readFileSync(path, 'utf-8'));
+  if (!data.perQuery || !Array.isArray(data.perQuery)) {
+    console.error(`Run JSON at ${path} has no perQuery array — re-run the bench with the updated harness.`);
+    process.exit(2);
+  }
+  return data;
+}
+
+const ours = loadRun(RUN_PATH);
+console.log(`# BEIR significance test (ADR-086)`);
+console.log(`Run: ${RUN_PATH}`);
+console.log(`Dataset: ${ours.dataset ?? '?'}  Model: ${ours.model ?? '?'}  Queries: ${ours.perQuery.length}`);
+console.log(`Bootstrap: ${ITERATIONS} iterations, seed=${SEED}`);
+
+const rng = mulberry32(SEED);
+const ourScores = ours.perQuery.map((q) => q.ndcg10);
+
+const t0 = performance.now();
+const oursCI = bootstrapCI(ourScores, ITERATIONS, rng);
+console.log(`\n=== Our nDCG@10 (1-sample bootstrap CI) ===`);
+console.log(`  point:    ${oursCI.point.toFixed(4)}`);
+console.log(`  95% CI:   [${oursCI.lower.toFixed(4)}, ${oursCI.upper.toFixed(4)}]`);
+
+if (ours.baselines) {
+  console.log(`\n=== vs each published baseline (CI overlap) ===`);
+  for (const [name, score] of Object.entries(ours.baselines)) {
+    const ciDiff = oursCI.point - score;
+    const lowerDiff = oursCI.lower - score;
+    const upperDiff = oursCI.upper - score;
+    const direction = ciDiff > 0 ? '↑ above' : '↓ below';
+    const sig = (lowerDiff > 0 && upperDiff > 0) ? 'p<0.05 (95% CI excludes baseline)'
+              : (lowerDiff < 0 && upperDiff < 0) ? 'p<0.05 (95% CI below baseline)'
+              : 'n.s. (95% CI overlaps baseline)';
+    console.log(`  ${score.toFixed(3)}  ${name.padEnd(28)}  Δ=${ciDiff >= 0 ? '+' : ''}${ciDiff.toFixed(4)}  ${direction}  [${sig}]`);
+  }
+}
+
+if (PAIRED_PATH) {
+  const baseline = loadRun(PAIRED_PATH);
+  console.log(`\n=== Paired bootstrap vs ${PAIRED_PATH} ===`);
+  const oursByQid = new Map(ours.perQuery.map((q) => [q.qid, q.ndcg10]));
+  const baseByQid = new Map(baseline.perQuery.map((q) => [q.qid, q.ndcg10]));
+  const rng2 = mulberry32(SEED + 1);
+  const paired = pairedBootstrap(oursByQid, baseByQid, ITERATIONS, rng2);
+  console.log(`  paired queries: ${paired.n_paired}`);
+  console.log(`  Δ (ours - baseline): ${paired.point_diff >= 0 ? '+' : ''}${paired.point_diff.toFixed(4)}`);
+  console.log(`  95% CI: [${paired.lower_95.toFixed(4)}, ${paired.upper_95.toFixed(4)}]`);
+  console.log(`  one-sided p (Δ ≤ 0): ${paired.p_one_sided.toFixed(4)}`);
+  const sig = paired.lower_95 > 0 ? 'SIGNIFICANT improvement (p<0.025 one-sided)' :
+              paired.upper_95 < 0 ? 'SIGNIFICANT regression (p<0.025 one-sided)' :
+              'NOT SIGNIFICANT — CI overlaps zero';
+  console.log(`  verdict: ${sig}`);
+}
+
+console.log(`\nBootstrap took ${((performance.now() - t0) / 1000).toFixed(1)}s.`);

package/scripts/run-beir-bge.mjs

@@ -26,7 +26,10 @@ const RUFLO_ROOT = resolve(SCRIPT_DIR, '../../../..');
 const RUNS_DIR = join(RUFLO_ROOT, 'docs', 'benchmarks', 'runs');
 
 const DATA_DIR = process.env.BEIR_DATA_DIR || '/tmp/beir-nfcorpus/nfcorpus';
-const CACHE_DIR = join('/tmp/beir-nfcorpus', 'bge-cache');
+// BUG-FIX (ADR-087): CACHE_DIR was hardcoded to /tmp/beir-nfcorpus/bge-cache,
+// which made the SciFact run silently overwrite the NFCorpus cache. Now
+// derived from DATA_DIR — each dataset gets its own cache directory.
+const CACHE_DIR = join(dirname(DATA_DIR), 'bge-cache');
 const BGE_MODEL = process.env.BGE_MODEL || 'Xenova/bge-base-en-v1.5';
 const SKIP_INGEST = process.env.SKIP_INGEST === '1';
 const MAX_QUERIES = Number(process.env.MAX_QUERIES) || 0;
@@ -218,25 +221,26 @@ async function main() {
 
   console.log(`\nRunning ${evalQueryIds.length} queries...`);
   let nSum = 0, mSum = 0, r10Sum = 0, r100Sum = 0, n = 0;
+  // ADR-086 — save per-query metrics for paired bootstrap significance testing.
+  const perQuery = [];
   const tQ = performance.now();
   for (const qid of evalQueryIds) {
     const qtext = queriesById.get(qid);
     if (!qtext) continue;
-    // BGE recommends prepending "Represent this sentence for searching relevant passages:" to queries
-    // but only for v1.5 *en-icl* variants. v1.5 (non-icl) works without it.
+    // BGE v1.5 non-icl works without the query prefix.
     const qEmb = await emb.embed(qtext);
-    // Score every doc, sort.
     const scores = new Array(docEmbeds.length);
     for (let i = 0; i < docEmbeds.length; i++) scores[i] = { id: docIds[i], score: cosine(qEmb, docEmbeds[i]) };
     scores.sort((a, b) => b.score - a.score);
     const top100 = scores.slice(0, 100).map((s) => s.id);
 
     const qmap = qrels.get(qid);
-    nSum += ndcg(top100, qmap, 10);
-    mSum += mrr(top100, qmap, 10);
-    r10Sum += recall(top100, qmap, 10);
-    r100Sum += recall(top100, qmap, 100);
-    n++;
+    const qNdcg = ndcg(top100, qmap, 10);
+    const qMrr = mrr(top100, qmap, 10);
+    const qR10 = recall(top100, qmap, 10);
+    const qR100 = recall(top100, qmap, 100);
+    nSum += qNdcg; mSum += qMrr; r10Sum += qR10; r100Sum += qR100; n++;
+    perQuery.push({ qid, ndcg10: qNdcg, mrr10: qMrr, recall10: qR10, recall100: qR100 });
     if (n % 50 === 0) {
       const elapsed = (performance.now() - tQ) / 1000;
       console.log(`  ${n}/${evalQueryIds.length} in ${elapsed.toFixed(0)}s  running nDCG@10=${(nSum / n).toFixed(4)}`);
@@ -276,6 +280,7 @@ async function main() {
     queries: n,
     corpusSize: corpus.length,
     metrics: { ndcg10, mrr10, recall10, recall100, avgQueryLatencyMs: queryMs / n },
+    perQuery, // ADR-086: per-query metrics for paired bootstrap significance testing
     baselines: BASELINES_NDCG10,
     ourRank,
     leaderboardLength: ranking.length,

package/scripts/run-beir-hybrid.mjs

@@ -0,0 +1,244 @@
+#!/usr/bin/env node
+// run-beir-hybrid.mjs — BM25 + dense (BGE) + RRF fusion + optional
+// cross-encoder rerank, evaluated on BEIR (ADR-087).
+//
+// Pipeline:
+//   1. BM25 over corpus (multi-field BM25 from hybrid-retrieval.ts)
+//   2. Dense cosine over BGE-base embeddings (cached from run-beir-bge.mjs)
+//   3. RRF fusion: score = sum over systems of 1/(k + rank) (k=60)
+//   4. Optional cross-encoder rerank of fused top-100 → top-10
+//
+// Reuses the .f32 doc-embedding cache from run-beir-bge.mjs — must run
+// that first to populate the cache.
+//
+// Usage:
+//   cd /tmp/beir-nfcorpus
+//   node /path/to/scripts/run-beir-hybrid.mjs                  # RRF only
+//   RERANK=1 node /path/to/scripts/run-beir-hybrid.mjs         # + cross-encoder rerank
+//   RRF_K=60 node /path/to/scripts/run-beir-hybrid.mjs         # tune RRF k
+//
+// ADR-087.
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join, resolve } from 'node:path';
+import { performance } from 'node:perf_hooks';
+
+const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
+const CLI_ROOT = resolve(SCRIPT_DIR, '..');
+const RUFLO_ROOT = resolve(SCRIPT_DIR, '../../../..');
+const RUNS_DIR = join(RUFLO_ROOT, 'docs', 'benchmarks', 'runs');
+
+const DATA_DIR = process.env.BEIR_DATA_DIR || '/tmp/beir-nfcorpus/nfcorpus';
+const BGE_MODEL = process.env.BGE_MODEL || 'Xenova/bge-base-en-v1.5';
+const CACHE_DIR = join(process.env.CACHE_BASE_DIR || dirname(DATA_DIR), 'bge-cache');
+const RRF_K = Number(process.env.RRF_K) || 60;
+const RERANK = process.env.RERANK === '1';
+const RERANK_TOP_K = Number(process.env.RERANK_TOP_K) || 100;
+const MAX_QUERIES = Number(process.env.MAX_QUERIES) || 0;
+
+// Same baselines table as run-beir-bge.mjs.
+const BASELINES_BY_DATASET = {
+  nfcorpus: { 'BM25 (Lucene)': 0.325, 'DocT5query': 0.328, 'TAS-B': 0.319, 'GenQ': 0.319, 'ColBERT': 0.305, 'Contriever': 0.328, 'GTR-XL': 0.343, 'SPLADE++': 0.347, 'BGE-large-v1.5 (pub)': 0.380, 'SBERT msmarco': 0.272 },
+  scifact:  { 'BM25 (Lucene)': 0.679, 'DocT5query': 0.675, 'TAS-B': 0.643, 'GenQ': 0.644, 'ColBERT': 0.671, 'Contriever': 0.677, 'GTR-XL': 0.662, 'SPLADE++': 0.704, 'BGE-large-v1.5 (pub)': 0.722, 'SBERT msmarco': 0.555 },
+};
+function detectDataset(path) {
+  const p = path.toLowerCase();
+  for (const ds of Object.keys(BASELINES_BY_DATASET)) if (p.includes(ds)) return ds;
+  return 'nfcorpus';
+}
+
+function dcg(rels, k) { let s = 0; for (let i = 0; i < Math.min(rels.length, k); i++) s += (Math.pow(2, rels[i]) - 1) / Math.log2(i + 2); return s; }
+function ndcg(retrieved, qrels, k) { const rels = retrieved.slice(0, k).map((id) => qrels.get(id) ?? 0); const ideal = [...qrels.values()].sort((a, b) => b - a).slice(0, k); const idcg = dcg(ideal, k); return idcg > 0 ? dcg(rels, k) / idcg : 0; }
+function mrr(retrieved, qrels, k) { for (let i = 0; i < Math.min(retrieved.length, k); i++) if ((qrels.get(retrieved[i]) ?? 0) > 0) return 1 / (i + 1); return 0; }
+function recall(retrieved, qrels, k) { const tot = [...qrels.values()].filter((v) => v > 0).length; if (tot === 0) return 0; let h = 0; for (let i = 0; i < Math.min(retrieved.length, k); i++) if ((qrels.get(retrieved[i]) ?? 0) > 0) h++; return h / tot; }
+function cosine(a, b) { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s; }
+
+function loadJsonl(path) { return readFileSync(path, 'utf-8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); }
+function loadQrels(path) { const q = new Map(); const lines = readFileSync(path, 'utf-8').split('\n'); for (let i = 1; i < lines.length; i++) { if (!lines[i].trim()) continue; const [qid, did, score] = lines[i].split('\t'); if (!q.has(qid)) q.set(qid, new Map()); q.get(qid).set(did, Number(score)); } return q; }
+
+function cachePath(suffix) { const safe = BGE_MODEL.replace(/\//g, '_'); return join(CACHE_DIR, `${safe}.${suffix}`); }
+function loadEmbeddings(dim) {
+  if (!existsSync(cachePath('ids')) || !existsSync(cachePath('f32'))) return null;
+  const ids = readFileSync(cachePath('ids'), 'utf-8').split('\n').filter(Boolean);
+  const raw = readFileSync(cachePath('f32'));
+  const buf = new Float32Array(raw.buffer, raw.byteOffset, raw.length / 4);
+  const embeds = [];
+  for (let i = 0; i < ids.length; i++) embeds.push(buf.slice(i * dim, (i + 1) * dim));
+  return { ids, embeds };
+}
+
+async function main() {
+  const dataset = detectDataset(DATA_DIR);
+  const BASELINES_NDCG10 = BASELINES_BY_DATASET[dataset];
+  console.log(`# BEIR ${dataset} — hybrid RRF${RERANK ? ' + cross-encoder rerank' : ''} (ADR-087)`);
+  console.log(`Data:  ${DATA_DIR}`);
+  console.log(`Dense: ${BGE_MODEL}`);
+  console.log(`RRF k: ${RRF_K}${RERANK ? `, rerank top-${RERANK_TOP_K}` : ''}`);
+
+  const corpus = loadJsonl(join(DATA_DIR, 'corpus.jsonl'));
+  const queries = loadJsonl(join(DATA_DIR, 'queries.jsonl'));
+  const qrels = loadQrels(join(DATA_DIR, 'qrels/test.tsv'));
+  console.log(`Corpus: ${corpus.length} docs · Test qrels: ${qrels.size}`);
+
+  // BGE embeddings — must be pre-cached from run-beir-bge.mjs.
+  const bge = await import(join(CLI_ROOT, 'dist/src/memory/bge-embedder.js'));
+  const emb = await bge.getBgeEmbedder(BGE_MODEL);
+  if (!emb) { console.error('BGE failed:', bge.getBgeStatus()); process.exit(1); }
+  const dim = emb.dim();
+  const cached = loadEmbeddings(dim);
+  if (!cached) { console.error(`No cached BGE embeddings at ${cachePath('f32')}. Run run-beir-bge.mjs first.`); process.exit(2); }
+  const docIds = cached.ids;
+  const docEmbeds = cached.embeds;
+  console.log(`Loaded ${docIds.length} cached BGE embeddings`);
+
+  // BM25 setup — multi-field over (title, text) per doc.
+  const { tokenize, buildCorpusStats, multiFieldBM25 } = await import(join(CLI_ROOT, 'dist/src/memory/hybrid-retrieval.js'));
+  const titleDocs = corpus.map((d) => tokenize(d.title || ''));
+  const textDocs = corpus.map((d) => tokenize(d.text || ''));
+  const titleStats = buildCorpusStats(titleDocs);
+  const textStats = buildCorpusStats(textDocs);
+  const SUBJECT_WEIGHT = Number(process.env.SUBJECT_WEIGHT ?? 1.0);  // titles in BEIR are short, neutral weight is fine
+  const BODY_WEIGHT = Number(process.env.BODY_WEIGHT ?? 1.0);
+  console.log(`BM25: multi-field (title sw=${SUBJECT_WEIGHT}, text bw=${BODY_WEIGHT})`);
+
+  // Optional cross-encoder reranker.
+  let crossEncoder = null;
+  if (RERANK) {
+    const ce = await import(join(CLI_ROOT, 'dist/src/memory/cross-encoder-rerank.js'));
+    const ceFn = await ce.getCrossEncoder('Xenova/ms-marco-MiniLM-L-6-v2');
+    if (!ceFn) { console.error('Cross-encoder failed to load:', ce.getCrossEncoderStatus()); process.exit(3); }
+    crossEncoder = ceFn;
+    console.log('Cross-encoder loaded: Xenova/ms-marco-MiniLM-L-6-v2');
+  }
+
+  const queriesById = new Map(queries.map((q) => [q._id, q.text]));
+  const evalQueryIds = MAX_QUERIES ? [...qrels.keys()].slice(0, MAX_QUERIES) : [...qrels.keys()];
+
+  // Build doc id → corpus index lookup (BM25 returns indices, dense returns ids).
+  const idToIdx = new Map(docIds.map((id, i) => [id, i]));
+  // Need a reverse map too: corpus index → doc id (for BM25 over corpus order).
+  const corpusIdToIdx = new Map(corpus.map((d, i) => [d._id, i]));
+  const corpusIdxToId = corpus.map((d) => d._id);
+
+  console.log(`\nRunning ${evalQueryIds.length} queries...`);
+  let nSum = 0, mSum = 0, r10Sum = 0, r100Sum = 0, n = 0;
+  const perQuery = [];
+  const tQ = performance.now();
+
+  for (const qid of evalQueryIds) {
+    const qtext = queriesById.get(qid);
+    if (!qtext) continue;
+
+    // §1 — dense BGE retrieval
+    const qEmb = await emb.embed(qtext);
+    const denseScored = new Array(docEmbeds.length);
+    for (let i = 0; i < docEmbeds.length; i++) denseScored[i] = { id: docIds[i], score: cosine(qEmb, docEmbeds[i]) };
+    denseScored.sort((a, b) => b.score - a.score);
+
+    // §2 — BM25 retrieval (over corpus order, then map to docIds)
+    const qTokens = tokenize(qtext);
+    const bm25Scored = new Array(corpus.length);
+    for (let i = 0; i < corpus.length; i++) {
+      const s = multiFieldBM25(qTokens, titleDocs[i], textDocs[i], titleStats, textStats, SUBJECT_WEIGHT, BODY_WEIGHT);
+      bm25Scored[i] = { id: corpusIdxToId[i], score: s };
+    }
+    bm25Scored.sort((a, b) => b.score - a.score);
+
+    // §3 — RRF fusion: score = sum over systems of 1/(k + rank).
+    // Only need top ~200 from each for stable RRF; cap at top-500 per system.
+    const TOP_PER_SYSTEM = 500;
+    const rrfScores = new Map();
+    for (let r = 0; r < Math.min(TOP_PER_SYSTEM, denseScored.length); r++) {
+      const id = denseScored[r].id;
+      rrfScores.set(id, (rrfScores.get(id) || 0) + 1 / (RRF_K + r + 1));
+    }
+    for (let r = 0; r < Math.min(TOP_PER_SYSTEM, bm25Scored.length); r++) {
+      const id = bm25Scored[r].id;
+      // Skip if this doc has 0 BM25 score (no token overlap) — true random docs would otherwise leak in
+      if (bm25Scored[r].score === 0) break;
+      rrfScores.set(id, (rrfScores.get(id) || 0) + 1 / (RRF_K + r + 1));
+    }
+    const fused = [...rrfScores.entries()]
+      .map(([id, score]) => ({ id, score }))
+      .sort((a, b) => b.score - a.score)
+      .slice(0, RERANK ? RERANK_TOP_K : 100);
+
+    let final = fused;
+
+    // §4 — optional cross-encoder rerank.
+    if (crossEncoder && fused.length > 0) {
+      const docsForRerank = fused.map(({ id }) => {
+        const ci = corpusIdToIdx.get(id);
+        const d = corpus[ci];
+        return `${d.title || ''} ${d.text || ''}`.slice(0, 4096);
+      });
+      const ceScores = await crossEncoder.scoreBatch(qtext, docsForRerank);
+      final = fused
+        .map((f, i) => ({ id: f.id, score: ceScores[i], rrf: f.score }))
+        .sort((a, b) => b.score - a.score);
+    }
+
+    const top100 = final.slice(0, 100).map((s) => s.id);
+    const qmap = qrels.get(qid);
+    const qNdcg = ndcg(top100, qmap, 10);
+    const qMrr = mrr(top100, qmap, 10);
+    const qR10 = recall(top100, qmap, 10);
+    const qR100 = recall(top100, qmap, 100);
+    nSum += qNdcg; mSum += qMrr; r10Sum += qR10; r100Sum += qR100; n++;
+    perQuery.push({ qid, ndcg10: qNdcg, mrr10: qMrr, recall10: qR10, recall100: qR100 });
+    if (n % 50 === 0) {
+      const elapsed = (performance.now() - tQ) / 1000;
+      console.log(`  ${n}/${evalQueryIds.length} in ${elapsed.toFixed(0)}s  running nDCG@10=${(nSum / n).toFixed(4)}`);
+    }
+  }
+  const queryMs = performance.now() - tQ;
+
+  const ndcg10 = nSum / n;
+  const mrr10 = mSum / n;
+  const recall10 = r10Sum / n;
+  const recall100 = r100Sum / n;
+
+  console.log(`\n=== Results (N=${n}, ${dataset}, BM25+BGE-base RRF k=${RRF_K}${RERANK ? ' + CE rerank' : ''}) ===`);
+  console.log(`  nDCG@10:    ${ndcg10.toFixed(4)}`);
+  console.log(`  MRR@10:     ${mrr10.toFixed(4)}`);
+  console.log(`  Recall@10:  ${recall10.toFixed(4)}`);
+  console.log(`  Recall@100: ${recall100.toFixed(4)}`);
+  console.log(`  Avg query latency: ${(queryMs / n).toFixed(0)}ms`);
+
+  const ourLabel = `ruflo + BM25+BGE-base RRF${RERANK ? '+CE' : ''}`;
+  const ranking = [
+    ...Object.entries(BASELINES_NDCG10).map(([name, score]) => ({ name, score, ours: false })),
+    { name: ourLabel, score: ndcg10, ours: true },
+  ].sort((a, b) => b.score - a.score);
+  console.log(`\n=== vs ${dataset} listed baselines ===`);
+  for (const r of ranking) console.log(`  ${r.score.toFixed(3)}  ${r.name}${r.ours ? ' ← us' : ''}`);
+  const ourRank = ranking.findIndex((r) => r.ours) + 1;
+  console.log(`\n  Our rank: ${ourRank} / ${ranking.length}`);
+
+  const summary = {
+    runAt: new Date().toISOString(),
+    benchmark: `beir-${dataset}-hybrid${RERANK ? '-rerank' : ''}`,
+    dataset,
+    pipeline: `BM25+BGE RRF${RERANK ? '+CE-rerank' : ''}`,
+    bgeModel: BGE_MODEL,
+    rrfK: RRF_K,
+    rerank: RERANK,
+    queries: n,
+    corpusSize: corpus.length,
+    metrics: { ndcg10, mrr10, recall10, recall100, avgQueryLatencyMs: queryMs / n },
+    perQuery,
+    baselines: BASELINES_NDCG10,
+    ourRank,
+    leaderboardLength: ranking.length,
+  };
+  mkdirSync(RUNS_DIR, { recursive: true });
+  const stamp = summary.runAt.replace(/[:.]/g, '-');
+  const suffix = RERANK ? 'hybrid-rerank' : 'hybrid-rrf';
+  writeFileSync(join(RUNS_DIR, `beir-${dataset}-${suffix}-${stamp}.json`), JSON.stringify(summary, null, 2));
+  writeFileSync(join(RUNS_DIR, `beir-${dataset}-${suffix}-latest.json`), JSON.stringify(summary, null, 2));
+  console.log(`\nWrote ${join(RUNS_DIR, `beir-${dataset}-${suffix}-${stamp}.json`)}`);
+  process.exit(0);
+}
+
+main().catch((err) => { console.error(err); process.exit(1); });

package/scripts/run-beir-rrf-ablation.mjs

@@ -0,0 +1,221 @@
+#!/usr/bin/env node
+// run-beir-rrf-ablation.mjs — RRF weight/k ablation matrix on a single
+// BEIR dataset (ADR-087).
+//
+// Pre-requisite: BGE doc embeddings cached at <dataset>/bge-cache/
+// (run scripts/run-beir-bge.mjs first).
+//
+// Reports:
+//   - Fixed default: k=60, equal weights (the headline number we ship)
+//   - Ablations: k=30/60/120 × weights (0.8/1.0, 1.0/1.0, 1.2/0.8 dense/bm25)
+//   - Bootstrap 95% CI on the fixed-default config only
+//
+// Usage:
+//   cd /tmp/beir-nfcorpus
+//   node /path/to/scripts/run-beir-rrf-ablation.mjs
+//   cd /tmp/beir-scifact
+//   BEIR_DATA_DIR=/tmp/beir-scifact/scifact node /path/to/scripts/run-beir-rrf-ablation.mjs
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join, resolve } from 'node:path';
+import { performance } from 'node:perf_hooks';
+
+const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
+const CLI_ROOT = resolve(SCRIPT_DIR, '..');
+const RUFLO_ROOT = resolve(SCRIPT_DIR, '../../../..');
+const RUNS_DIR = join(RUFLO_ROOT, 'docs', 'benchmarks', 'runs');
+
+const DATA_DIR = process.env.BEIR_DATA_DIR || '/tmp/beir-nfcorpus/nfcorpus';
+const BGE_MODEL = process.env.BGE_MODEL || 'Xenova/bge-base-en-v1.5';
+const CACHE_DIR = join(dirname(DATA_DIR), 'bge-cache');
+
+const BASELINES = {
+  nfcorpus: { 'BM25': 0.325, 'BGE-large-pub': 0.380, 'SPLADE++': 0.347 },
+  scifact:  { 'BM25': 0.679, 'BGE-large-pub': 0.722, 'SPLADE++': 0.704 },
+};
+function detectDataset(p) { const x = p.toLowerCase(); return Object.keys(BASELINES).find(d => x.includes(d)) || 'nfcorpus'; }
+const DATASET = detectDataset(DATA_DIR);
+
+function dcg(rels, k) { let s = 0; for (let i = 0; i < Math.min(rels.length, k); i++) s += (Math.pow(2, rels[i]) - 1) / Math.log2(i + 2); return s; }
+function ndcg(retrieved, qrels, k) { const rels = retrieved.slice(0, k).map((id) => qrels.get(id) ?? 0); const ideal = [...qrels.values()].sort((a, b) => b - a).slice(0, k); const idcg = dcg(ideal, k); return idcg > 0 ? dcg(rels, k) / idcg : 0; }
+function recall(retrieved, qrels, k) { const tot = [...qrels.values()].filter((v) => v > 0).length; if (tot === 0) return 0; let h = 0; for (let i = 0; i < Math.min(retrieved.length, k); i++) if ((qrels.get(retrieved[i]) ?? 0) > 0) h++; return h / tot; }
+function cosine(a, b) { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s; }
+
+function loadJsonl(p) { return readFileSync(p, 'utf-8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); }
+function loadQrels(p) { const q = new Map(); const lines = readFileSync(p, 'utf-8').split('\n'); for (let i = 1; i < lines.length; i++) { if (!lines[i].trim()) continue; const [qid, did, s] = lines[i].split('\t'); if (!q.has(qid)) q.set(qid, new Map()); q.get(qid).set(did, Number(s)); } return q; }
+
+function cachePath(s) { const safe = BGE_MODEL.replace(/\//g, '_'); return join(CACHE_DIR, `${safe}.${s}`); }
+function loadEmbeddings(dim) {
+  if (!existsSync(cachePath('ids')) || !existsSync(cachePath('f32'))) return null;
+  const ids = readFileSync(cachePath('ids'), 'utf-8').split('\n').filter(Boolean);
+  const raw = readFileSync(cachePath('f32'));
+  const buf = new Float32Array(raw.buffer, raw.byteOffset, raw.length / 4);
+  return { ids, embeds: ids.map((_, i) => buf.slice(i * dim, (i + 1) * dim)) };
+}
+
+// RRF — pure function per the original Cormack/Clarke/Buettcher 2009 paper.
+// `rankings` is an array of [{id, score}, ...] sorted descending by score.
+// `weights` is per-system weight (default 1.0 each).
+function rrfFuse(rankings, k = 60, weights) {
+  const scores = new Map();
+  rankings.forEach((ranking, sIdx) => {
+    const w = weights?.[sIdx] ?? 1;
+    ranking.forEach((doc, rankIdx) => {
+      const rank = rankIdx + 1;
+      scores.set(doc.id, (scores.get(doc.id) ?? 0) + w / (k + rank));
+    });
+  });
+  return [...scores.entries()].map(([id, score]) => ({ id, score })).sort((a, b) => b.score - a.score);
+}
+
+// Mulberry32 PRNG for deterministic bootstrap.
+function mulberry32(a) { return function () { a |= 0; a = (a + 0x6D2B79F5) | 0; let t = a; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; }; }
+function bootstrapCI(scores, iter = 10000, rng = mulberry32(42)) {
+  const n = scores.length; const means = new Float64Array(iter);
+  for (let i = 0; i < iter; i++) { let s = 0; for (let j = 0; j < n; j++) s += scores[Math.floor(rng() * n)]; means[i] = s / n; }
+  const sorted = Array.from(means).sort((a, b) => a - b);
+  const lo = sorted[Math.floor(0.025 * iter)]; const hi = sorted[Math.floor(0.975 * iter)];
+  return { mean: scores.reduce((s, v) => s + v, 0) / n, lo, hi };
+}
+
+async function main() {
+  console.log(`# RRF ablation on BEIR ${DATASET} (ADR-087)`);
+  console.log(`Data: ${DATA_DIR}\nDense: ${BGE_MODEL}\n`);
+
+  const corpus = loadJsonl(join(DATA_DIR, 'corpus.jsonl'));
+  const queries = loadJsonl(join(DATA_DIR, 'queries.jsonl'));
+  const qrels = loadQrels(join(DATA_DIR, 'qrels/test.tsv'));
+  console.log(`Corpus: ${corpus.length} · queries: ${queries.length} · test qrels: ${qrels.size}`);
+
+  // Dense
+  const bge = await import(join(CLI_ROOT, 'dist/src/memory/bge-embedder.js'));
+  const emb = await bge.getBgeEmbedder(BGE_MODEL);
+  if (!emb) { console.error('BGE failed'); process.exit(1); }
+  const cached = loadEmbeddings(emb.dim());
+  if (!cached) { console.error(`No cached embeddings at ${cachePath('f32')}. Run run-beir-bge.mjs first.`); process.exit(2); }
+  const { ids: docIds, embeds: docEmbeds } = cached;
+
+  // BM25
+  const { tokenize, buildCorpusStats, multiFieldBM25 } = await import(join(CLI_ROOT, 'dist/src/memory/hybrid-retrieval.js'));
+  const titleDocs = corpus.map((d) => tokenize(d.title || ''));
+  const textDocs = corpus.map((d) => tokenize(d.text || ''));
+  const titleStats = buildCorpusStats(titleDocs);
+  const textStats = buildCorpusStats(textDocs);
+  const corpusIdxToId = corpus.map((d) => d._id);
+
+  // Pre-compute per-query rankings for both systems ONCE, so the ablation
+  // matrix below just permutes RRF k/weights, not the underlying retrievals.
+  const queriesById = new Map(queries.map((q) => [q._id, q.text]));
+  const evalQids = [...qrels.keys()];
+
+  console.log(`Pre-computing dense + BM25 rankings for ${evalQids.length} queries...`);
+  const cached_dense = new Map();
+  const cached_bm25 = new Map();
+  const TOP_K_RETRIEVE = 1000;  // enough for stable RRF
+  const tPre = performance.now();
+  for (const qid of evalQids) {
+    const qtext = queriesById.get(qid);
+    if (!qtext) continue;
+    // Dense
+    const qEmb = await emb.embed(qtext);
+    const dense = docIds.map((id, i) => ({ id, score: cosine(qEmb, docEmbeds[i]) })).sort((a, b) => b.score - a.score).slice(0, TOP_K_RETRIEVE);
+    cached_dense.set(qid, dense);
+    // BM25
+    const qTokens = tokenize(qtext);
+    const bm25 = corpus.map((d, i) => ({ id: corpusIdxToId[i], score: multiFieldBM25(qTokens, titleDocs[i], textDocs[i], titleStats, textStats, 1, 1) })).filter((d) => d.score > 0).sort((a, b) => b.score - a.score).slice(0, TOP_K_RETRIEVE);
+    cached_bm25.set(qid, bm25);
+  }
+  console.log(`Pre-compute done in ${((performance.now() - tPre) / 1000).toFixed(0)}s.\n`);
+
+  // ----- ABLATION GRID -----
+  // FIXED-DEFAULT first (the ship claim). Other variants reported as exploratory.
+  const CONFIGS = [
+    // The headline configurations — defaults fixed before viewing results.
+    { name: 'dense only',             rrf: false, weights: null,           kVal: null },
+    { name: 'BM25 only',              rrf: false, weights: null,           kVal: null, bm25Only: true },
+    { name: 'RRF k=60 equal (default)', rrf: true,  weights: [1.0, 1.0],     kVal: 60 },
+    // Exploratory — disclose as tuned variants, not default.
+    { name: 'RRF k=30 equal',         rrf: true,  weights: [1.0, 1.0],     kVal: 30 },
+    { name: 'RRF k=120 equal',        rrf: true,  weights: [1.0, 1.0],     kVal: 120 },
+    { name: 'RRF k=60 dense=1.2,bm25=0.8', rrf: true,  weights: [1.2, 0.8],     kVal: 60 },
+    { name: 'RRF k=60 dense=0.8,bm25=1.2', rrf: true,  weights: [0.8, 1.2],     kVal: 60 },
+  ];
+
+  function scoreConfig(cfg) {
+    let nSum = 0, r10Sum = 0, r100Sum = 0, n = 0;
+    const perQuery = [];
+    for (const qid of evalQids) {
+      const dense = cached_dense.get(qid);
+      const bm25 = cached_bm25.get(qid);
+      if (!dense) continue;
+      let top100;
+      if (!cfg.rrf) {
+        if (cfg.bm25Only) top100 = (bm25 || []).slice(0, 100).map((d) => d.id);
+        else top100 = dense.slice(0, 100).map((d) => d.id);
+      } else {
+        const fused = rrfFuse([dense, bm25 || []], cfg.kVal, cfg.weights);
+        top100 = fused.slice(0, 100).map((d) => d.id);
+      }
+      const qmap = qrels.get(qid);
+      const ndcg10 = ndcg(top100, qmap, 10);
+      nSum += ndcg10;
+      r10Sum += recall(top100, qmap, 10);
+      r100Sum += recall(top100, qmap, 100);
+      n++;
+      perQuery.push({ qid, ndcg10 });
+    }
+    return {
+      name: cfg.name,
+      ndcg10: nSum / n,
+      recall10: r10Sum / n,
+      recall100: r100Sum / n,
+      n,
+      perQuery,
+    };
+  }
+
+  console.log('Config                                | nDCG@10  R@10    R@100   95% CI');
+  console.log('--------------------------------------|---------------------------------');
+  const results = [];
+  let defaultResult = null;
+  for (const cfg of CONFIGS) {
+    const r = scoreConfig(cfg);
+    const ci = bootstrapCI(r.perQuery.map((q) => q.ndcg10));
+    const ciStr = `[${ci.lo.toFixed(3)}, ${ci.hi.toFixed(3)}]`;
+    console.log(`${cfg.name.padEnd(38)} | ${r.ndcg10.toFixed(4)}  ${r.recall10.toFixed(3)}   ${r.recall100.toFixed(3)}   ${ciStr}`);
+    results.push({ ...r, ci, cfg });
+    if (cfg.name.includes('default')) defaultResult = { ...r, ci, cfg };
+  }
+
+  console.log('\n=== Ship claim (fixed default, k=60, equal weights) ===');
+  console.log(`Dataset: ${DATASET}`);
+  console.log(`nDCG@10: ${defaultResult.ndcg10.toFixed(4)}   95% CI ${`[${defaultResult.ci.lo.toFixed(4)}, ${defaultResult.ci.hi.toFixed(4)}]`}`);
+  for (const [name, score] of Object.entries(BASELINES[DATASET])) {
+    const delta = defaultResult.ndcg10 - score;
+    const inCi = score >= defaultResult.ci.lo && score <= defaultResult.ci.hi;
+    const sig = inCi ? 'n.s.' : (delta > 0 ? 'p<0.05 WIN' : 'p<0.05 LOSS');
+    console.log(`  vs ${name.padEnd(15)} Δ=${(delta >= 0 ? '+' : '') + delta.toFixed(4)}   ${sig}`);
+  }
+
+  // Save
+  const summary = {
+    runAt: new Date().toISOString(),
+    benchmark: `beir-${DATASET}-rrf-ablation`,
+    dataset: DATASET,
+    bgeModel: BGE_MODEL,
+    queries: defaultResult.n,
+    corpusSize: corpus.length,
+    fixedDefault: { name: defaultResult.cfg.name, metrics: { ndcg10: defaultResult.ndcg10, recall10: defaultResult.recall10, recall100: defaultResult.recall100 }, ci: defaultResult.ci, perQuery: defaultResult.perQuery },
+    ablation: results.map((r) => ({ name: r.cfg.name, ndcg10: r.ndcg10, recall10: r.recall10, recall100: r.recall100, ci: r.ci })),
+    baselines: BASELINES[DATASET],
+  };
+  mkdirSync(RUNS_DIR, { recursive: true });
+  const stamp = summary.runAt.replace(/[:.]/g, '-');
+  writeFileSync(join(RUNS_DIR, `beir-${DATASET}-rrf-ablation-${stamp}.json`), JSON.stringify(summary, null, 2));
+  writeFileSync(join(RUNS_DIR, `beir-${DATASET}-rrf-ablation-latest.json`), JSON.stringify(summary, null, 2));
+  console.log(`\nWrote ${join(RUNS_DIR, `beir-${DATASET}-rrf-ablation-${stamp}.json`)}`);
+  process.exit(0);
+}
+
+main().catch((err) => { console.error(err); process.exit(1); });