@claude-flow/cli 3.21.1 → 3.22.0

package/scripts/run-beir-lucene-bm25.mjs

@@ -1,154 +0,0 @@
-#!/usr/bin/env node
-// run-beir-lucene-bm25.mjs — test our Lucene-style BM25 against the
-// published BEIR BM25 baselines. Also runs RRF with Lucene-BM25 + dense
-// to see if the stronger BM25 closes the asymmetric-strength gap that
-// broke RRF in ADR-087.
-//
-// Usage:
-//   cd /tmp/beir-nfcorpus
-//   node /path/to/scripts/run-beir-lucene-bm25.mjs
-//   BEIR_DATA_DIR=/tmp/beir-scifact/scifact node /path/to/scripts/run-beir-lucene-bm25.mjs
-
-import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
-import { fileURLToPath } from 'node:url';
-import { dirname, join, resolve } from 'node:path';
-import { performance } from 'node:perf_hooks';
-
-const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
-const CLI_ROOT = resolve(SCRIPT_DIR, '..');
-const RUFLO_ROOT = resolve(SCRIPT_DIR, '../../../..');
-const RUNS_DIR = join(RUFLO_ROOT, 'docs', 'benchmarks', 'runs');
-
-const DATA_DIR = process.env.BEIR_DATA_DIR || '/tmp/beir-nfcorpus/nfcorpus';
-const BGE_MODEL = 'Xenova/bge-base-en-v1.5';
-const CACHE_DIR = join(dirname(DATA_DIR), 'bge-cache');
-
-const BASELINES = {
-  nfcorpus: { BM25: 0.325, 'BGE-large-pub': 0.380, 'SPLADE++': 0.347 },
-  scifact: { BM25: 0.679, 'BGE-large-pub': 0.722, 'SPLADE++': 0.704 },
-};
-function detect(p) { const x = p.toLowerCase(); return Object.keys(BASELINES).find(d => x.includes(d)) || 'nfcorpus'; }
-
-function dcg(rels, k) { let s = 0; for (let i = 0; i < Math.min(rels.length, k); i++) s += (Math.pow(2, rels[i]) - 1) / Math.log2(i + 2); return s; }
-function ndcg(retrieved, qrels, k) { const rels = retrieved.slice(0, k).map((id) => qrels.get(id) ?? 0); const ideal = [...qrels.values()].sort((a, b) => b - a).slice(0, k); const idcg = dcg(ideal, k); return idcg > 0 ? dcg(rels, k) / idcg : 0; }
-function recall(retrieved, qrels, k) { const tot = [...qrels.values()].filter((v) => v > 0).length; if (tot === 0) return 0; let h = 0; for (let i = 0; i < Math.min(retrieved.length, k); i++) if ((qrels.get(retrieved[i]) ?? 0) > 0) h++; return h / tot; }
-function cosine(a, b) { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s; }
-
-function loadJsonl(p) { return readFileSync(p, 'utf-8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); }
-function loadQrels(p) { const q = new Map(); const lines = readFileSync(p, 'utf-8').split('\n'); for (let i = 1; i < lines.length; i++) { if (!lines[i].trim()) continue; const [qid, did, s] = lines[i].split('\t'); if (!q.has(qid)) q.set(qid, new Map()); q.get(qid).set(did, Number(s)); } return q; }
-
-function cachePath(s) { const safe = BGE_MODEL.replace(/\//g, '_'); return join(CACHE_DIR, `${safe}.${s}`); }
-function loadEmbeddings(dim) {
-  if (!existsSync(cachePath('ids')) || !existsSync(cachePath('f32'))) return null;
-  const ids = readFileSync(cachePath('ids'), 'utf-8').split('\n').filter(Boolean);
-  const raw = readFileSync(cachePath('f32'));
-  const buf = new Float32Array(raw.buffer, raw.byteOffset, raw.length / 4);
-  return { ids, embeds: ids.map((_, i) => buf.slice(i * dim, (i + 1) * dim)) };
-}
-
-function rrfFuse(rankings, k = 60, weights) {
-  const scores = new Map();
-  rankings.forEach((ranking, sIdx) => {
-    const w = weights?.[sIdx] ?? 1;
-    ranking.forEach((doc, rankIdx) => {
-      const rank = rankIdx + 1;
-      scores.set(doc.id, (scores.get(doc.id) ?? 0) + w / (k + rank));
-    });
-  });
-  return [...scores.entries()].map(([id, score]) => ({ id, score })).sort((a, b) => b.score - a.score);
-}
-
-async function main() {
-  const dataset = detect(DATA_DIR);
-  console.log(`# Lucene-style BM25 + RRF on BEIR ${dataset} (ADR-088)`);
-  console.log(`Data: ${DATA_DIR}\n`);
-
-  const corpus = loadJsonl(join(DATA_DIR, 'corpus.jsonl'));
-  const queries = loadJsonl(join(DATA_DIR, 'queries.jsonl'));
-  const qrels = loadQrels(join(DATA_DIR, 'qrels/test.tsv'));
-  console.log(`Corpus: ${corpus.length} · test qrels: ${qrels.size}`);
-
-  // Lucene BM25
-  const { luceneTokenize, buildLuceneCorpusStats, luceneBM25 } = await import(join(CLI_ROOT, 'dist/src/memory/lucene-bm25.js'));
-  console.log('\nTokenising corpus (Lucene-style: Porter stem + Lucene stopwords + length norm)...');
-  const tIngest = performance.now();
-  const docs = corpus.map((d) => luceneTokenize(`${d.title || ''} ${d.text || ''}`));
-  const stats = buildLuceneCorpusStats(docs);
-  console.log(`Tokenized + stats built in ${((performance.now() - tIngest) / 1000).toFixed(1)}s. avgDocLen=${stats.avgDocLen.toFixed(0)}, vocab=${stats.idf.size}`);
-
-  // BGE dense (from cache)
-  const bge = await import(join(CLI_ROOT, 'dist/src/memory/bge-embedder.js'));
-  const emb = await bge.getBgeEmbedder(BGE_MODEL);
-  if (!emb) { console.error('BGE failed'); process.exit(1); }
-  const cached = loadEmbeddings(emb.dim());
-  if (!cached) { console.error(`No cached embeddings. Run run-beir-bge.mjs first.`); process.exit(2); }
-  const { ids: docIds, embeds: docEmbeds } = cached;
-  const corpusIdxToId = corpus.map((d) => d._id);
-
-  // Run all configs
-  const queriesById = new Map(queries.map((q) => [q._id, q.text]));
-  const evalQids = [...qrels.keys()];
-  console.log(`\nRunning ${evalQids.length} queries through 4 configs (dense, Lucene-BM25, RRF, RRF-tuned)...\n`);
-
-  let n = 0;
-  const results = {
-    dense: { n: 0, ndcg: 0, r10: 0, r100: 0 },
-    bm25: { n: 0, ndcg: 0, r10: 0, r100: 0 },
-    rrf60: { n: 0, ndcg: 0, r10: 0, r100: 0 },
-    rrf30: { n: 0, ndcg: 0, r10: 0, r100: 0 },
-  };
-  const tQ = performance.now();
-  for (const qid of evalQids) {
-    const qtext = queriesById.get(qid);
-    if (!qtext) continue;
-    n++;
-
-    // Dense
-    const qEmb = await emb.embed(qtext);
-    const dense = docIds.map((id, i) => ({ id, score: cosine(qEmb, docEmbeds[i]) })).sort((a, b) => b.score - a.score).slice(0, 1000);
-
-    // BM25
-    const qTokens = luceneTokenize(qtext);
-    const bm25 = corpus.map((d, i) => ({ id: corpusIdxToId[i], score: luceneBM25(qTokens, docs[i], stats) })).filter((d) => d.score > 0).sort((a, b) => b.score - a.score).slice(0, 1000);
-
-    // RRF k=60 and k=30
-    const rrf60 = rrfFuse([dense, bm25], 60);
-    const rrf30 = rrfFuse([dense, bm25], 30);
-
-    const qmap = qrels.get(qid);
-    for (const [key, top] of [['dense', dense], ['bm25', bm25], ['rrf60', rrf60], ['rrf30', rrf30]]) {
-      const top100 = top.slice(0, 100).map((s) => s.id);
-      results[key].n++;
-      results[key].ndcg += ndcg(top100, qmap, 10);
-      results[key].r10 += recall(top100, qmap, 10);
-      results[key].r100 += recall(top100, qmap, 100);
-    }
-    if (n % 50 === 0) console.log(`  ${n}/${evalQids.length} dense=${(results.dense.ndcg / results.dense.n).toFixed(3)} bm25=${(results.bm25.ndcg / results.bm25.n).toFixed(3)} rrf60=${(results.rrf60.ndcg / results.rrf60.n).toFixed(3)}`);
-  }
-  const elapsed = (performance.now() - tQ) / 1000;
-
-  console.log(`\nDone in ${elapsed.toFixed(0)}s.\n`);
-  console.log('Config         | nDCG@10  R@10   R@100  vs published-BM25');
-  console.log('---------------|---------------------------------');
-  for (const [key, r] of Object.entries(results)) {
-    const score = r.ndcg / r.n;
-    const delta = score - BASELINES[dataset].BM25;
-    console.log(`${key.padEnd(14)} | ${score.toFixed(4)}  ${(r.r10 / r.n).toFixed(3)}  ${(r.r100 / r.n).toFixed(3)}  ${(delta >= 0 ? '+' : '') + delta.toFixed(4)}`);
-  }
-
-  const summary = {
-    runAt: new Date().toISOString(),
-    benchmark: `beir-${dataset}-lucene-bm25`,
-    dataset, model: BGE_MODEL, queries: n,
-    results: Object.fromEntries(Object.entries(results).map(([k, v]) => [k, { ndcg10: v.ndcg / v.n, recall10: v.r10 / v.n, recall100: v.r100 / v.n }])),
-    baselines: BASELINES[dataset],
-  };
-  mkdirSync(RUNS_DIR, { recursive: true });
-  const stamp = summary.runAt.replace(/[:.]/g, '-');
-  writeFileSync(join(RUNS_DIR, `beir-${dataset}-lucene-bm25-${stamp}.json`), JSON.stringify(summary, null, 2));
-  writeFileSync(join(RUNS_DIR, `beir-${dataset}-lucene-bm25-latest.json`), JSON.stringify(summary, null, 2));
-  console.log(`\nWrote ${join(RUNS_DIR, `beir-${dataset}-lucene-bm25-${stamp}.json`)}`);
-  process.exit(0);
-}
-
-main().catch((err) => { console.error(err); process.exit(1); });

package/scripts/run-beir-nfcorpus.mjs

@@ -1,231 +0,0 @@
-#!/usr/bin/env node
-// run-beir-nfcorpus.mjs — public benchmark harness for BEIR NFCorpus.
-//
-// Why NFCorpus: smallest BEIR dataset (3.6k docs, 323 test queries) with a
-// well-published BM25 baseline and dense-retriever ceilings — runs end-to-end
-// in <15 min on this hardware.
-//
-// Setup:
-//   mkdir -p /tmp/beir-nfcorpus && cd /tmp/beir-nfcorpus
-//   curl -sL -o nfcorpus.zip 'https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nfcorpus.zip'
-//   unzip -q nfcorpus.zip
-//
-// Run:
-//   cd /tmp/beir-nfcorpus && rm -rf .claude-flow
-//   node /path/to/scripts/run-beir-nfcorpus.mjs
-//
-// Reports:
-//   - nDCG@10 (graded, BEIR standard)
-//   - MRR@10
-//   - Recall@10, Recall@100
-//   - Published comparisons (BM25, ColBERT, SPLADE, etc.)
-
-import { readFileSync, writeFileSync, mkdirSync, readdirSync, statSync } from 'node:fs';
-import { fileURLToPath } from 'node:url';
-import { dirname, join, resolve } from 'node:path';
-import { performance } from 'node:perf_hooks';
-
-const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
-const CLI_ROOT = resolve(SCRIPT_DIR, '..');
-const RUFLO_ROOT = resolve(SCRIPT_DIR, '../../../..');
-const RUNS_DIR = join(RUFLO_ROOT, 'docs', 'benchmarks', 'runs');
-
-const DATA_DIR = process.env.BEIR_DATA_DIR || '/tmp/beir-nfcorpus/nfcorpus';
-const TOP_K = Number(process.env.TOP_K) || 100;     // top-100 for recall@100
-const NDCG_K = 10;
-const MAX_QUERIES = Number(process.env.MAX_QUERIES) || 0;  // 0 = all
-const SKIP_INGEST = process.env.SKIP_INGEST === '1'; // reuse existing store
-
-// Published BEIR NFCorpus baselines (from Thakur et al. 2021 + leaderboard tracker)
-const BASELINES_NDCG10 = {
-  'BM25':            0.325,
-  'DocT5query':      0.328,
-  'SBERT (msmarco)': 0.272,  // dense bi-encoder on MS MARCO
-  'TAS-B':           0.319,
-  'GenQ':            0.319,
-  'ColBERT':         0.305,
-  'SPLADE++':        0.347,  // top dense at time of paper
-  'Contriever':      0.328,
-  'GTR-XL':          0.343,
-  'BGE-large-v1.5':  0.380,  // current ~top-of-class
-};
-
-// ---------------------------------------------------------------------------
-// nDCG with graded relevance (BEIR standard)
-// ---------------------------------------------------------------------------
-
-function dcg(rels, k) {
-  let s = 0;
-  for (let i = 0; i < Math.min(rels.length, k); i++) {
-    s += (Math.pow(2, rels[i]) - 1) / Math.log2(i + 2);
-  }
-  return s;
-}
-
-function ndcg(retrievedDocIds, qrelsMap, k) {
-  const rels = retrievedDocIds.slice(0, k).map((id) => qrelsMap.get(id) ?? 0);
-  const ideal = [...qrelsMap.values()].sort((a, b) => b - a).slice(0, k);
-  const idcg = dcg(ideal, k);
-  return idcg > 0 ? dcg(rels, k) / idcg : 0;
-}
-
-function mrr(retrievedDocIds, qrelsMap, k) {
-  for (let i = 0; i < Math.min(retrievedDocIds.length, k); i++) {
-    if ((qrelsMap.get(retrievedDocIds[i]) ?? 0) > 0) return 1 / (i + 1);
-  }
-  return 0;
-}
-
-function recall(retrievedDocIds, qrelsMap, k) {
-  const totalRelevant = [...qrelsMap.values()].filter((v) => v > 0).length;
-  if (totalRelevant === 0) return 0;
-  let hit = 0;
-  for (let i = 0; i < Math.min(retrievedDocIds.length, k); i++) {
-    if ((qrelsMap.get(retrievedDocIds[i]) ?? 0) > 0) hit++;
-  }
-  return hit / totalRelevant;
-}
-
-// ---------------------------------------------------------------------------
-// Loaders
-// ---------------------------------------------------------------------------
-
-function loadJsonl(path) {
-  const items = [];
-  for (const line of readFileSync(path, 'utf-8').split('\n')) {
-    if (!line.trim()) continue;
-    items.push(JSON.parse(line));
-  }
-  return items;
-}
-
-function loadQrels(path) {
-  // Map<queryId, Map<docId, grade>>
-  const qrels = new Map();
-  const lines = readFileSync(path, 'utf-8').split('\n');
-  for (let i = 1; i < lines.length; i++) {
-    if (!lines[i].trim()) continue;
-    const [qid, did, score] = lines[i].split('\t');
-    if (!qrels.has(qid)) qrels.set(qid, new Map());
-    qrels.get(qid).set(did, Number(score));
-  }
-  return qrels;
-}
-
-// ---------------------------------------------------------------------------
-// Main
-// ---------------------------------------------------------------------------
-
-async function main() {
-  console.log(`# BEIR NFCorpus benchmark (ADR-085)`);
-  console.log(`Data: ${DATA_DIR}`);
-  console.log(`Cwd: ${process.cwd()}`);
-
-  const corpus = loadJsonl(join(DATA_DIR, 'corpus.jsonl'));
-  const queries = loadJsonl(join(DATA_DIR, 'queries.jsonl'));
-  const qrels = loadQrels(join(DATA_DIR, 'qrels/test.tsv'));
-
-  console.log(`Corpus: ${corpus.length} docs, ${queries.length} queries (total), ${qrels.size} eval queries (test split)`);
-
-  const neural = await import(join(CLI_ROOT, 'dist/src/mcp-tools/neural-tools.js'));
-  const tool = neural.neuralTools.find((t) => t.name === 'neural_patterns');
-
-  // §1 — Ingest corpus (id-stable pattern name = doc _id; content = title + text).
-  // Skip if SKIP_INGEST=1 and store already populated.
-  if (!SKIP_INGEST) {
-    console.log(`\nIngesting ${corpus.length} docs...`);
-    const t0 = performance.now();
-    // Batch in chunks to avoid memory pressure.
-    const CHUNK = 200;
-    for (let i = 0; i < corpus.length; i += CHUNK) {
-      const batch = corpus.slice(i, i + CHUNK);
-      await neural.storeNeuralPatterns(batch.map((d) => ({
-        name: d._id,                         // use _id as the stable identifier
-        type: 'beir-nfcorpus',
-        content: `${d.title || ''}\n${d.text || ''}`.slice(0, 4096),
-      })));
-      if ((i + CHUNK) % 1000 === 0 || i + CHUNK >= corpus.length) {
-        const pct = Math.min(100, ((i + CHUNK) / corpus.length * 100)).toFixed(0);
-        const elapsed = (performance.now() - t0) / 1000;
-        console.log(`  ${Math.min(i + CHUNK, corpus.length)}/${corpus.length} (${pct}%) in ${elapsed.toFixed(0)}s`);
-      }
-    }
-    console.log(`Ingest done in ${((performance.now() - t0) / 1000).toFixed(0)}s`);
-  } else {
-    console.log('SKIP_INGEST=1 → reusing existing store');
-  }
-
-  // §2 — Build query lookup (eval queries only).
-  const queriesById = new Map(queries.map((q) => [q._id, q.text]));
-  const evalQueryIds = MAX_QUERIES
-    ? [...qrels.keys()].slice(0, MAX_QUERIES)
-    : [...qrels.keys()];
-
-  // §3 — Run retrieval for each query, compute metrics.
-  console.log(`\nRunning ${evalQueryIds.length} queries...`);
-  let ndcgSum = 0, mrrSum = 0, recall10Sum = 0, recall100Sum = 0;
-  const tQ = performance.now();
-  let queryIdx = 0;
-  for (const qid of evalQueryIds) {
-    const qtext = queriesById.get(qid);
-    if (!qtext) { queryIdx++; continue; }
-    const r = await tool.handler({ action: 'search', query: qtext, mode: 'hybrid', limit: TOP_K, rerank: false });
-    const retrievedIds = (r.results || []).map((m) => m.name);  // we set name = _id
-    const qrelsMap = qrels.get(qid) || new Map();
-    ndcgSum += ndcg(retrievedIds, qrelsMap, NDCG_K);
-    mrrSum += mrr(retrievedIds, qrelsMap, NDCG_K);
-    recall10Sum += recall(retrievedIds, qrelsMap, 10);
-    recall100Sum += recall(retrievedIds, qrelsMap, 100);
-    queryIdx++;
-    if (queryIdx % 50 === 0) {
-      const elapsed = (performance.now() - tQ) / 1000;
-      console.log(`  ${queryIdx}/${evalQueryIds.length} in ${elapsed.toFixed(0)}s — running nDCG@10=${(ndcgSum / queryIdx).toFixed(4)}`);
-    }
-  }
-  const queryMs = performance.now() - tQ;
-  const N = evalQueryIds.length;
-  const ndcg10 = ndcgSum / N;
-  const mrr10 = mrrSum / N;
-  const recall10 = recall10Sum / N;
-  const recall100 = recall100Sum / N;
-
-  console.log(`\n=== Results (N=${N} queries) ===`);
-  console.log(`  nDCG@10:    ${ndcg10.toFixed(4)}`);
-  console.log(`  MRR@10:     ${mrr10.toFixed(4)}`);
-  console.log(`  Recall@10:  ${recall10.toFixed(4)}`);
-  console.log(`  Recall@100: ${recall100.toFixed(4)}`);
-  console.log(`  Avg query latency: ${(queryMs / N).toFixed(0)}ms`);
-
-  console.log(`\n=== vs published NFCorpus nDCG@10 baselines ===`);
-  const ourEntry = { name: 'ruflo hybrid (3.10.25)', score: ndcg10, ours: true };
-  const ranking = [
-    ...Object.entries(BASELINES_NDCG10).map(([name, score]) => ({ name, score, ours: false })),
-    ourEntry,
-  ].sort((a, b) => b.score - a.score);
-  for (const r of ranking) {
-    const marker = r.ours ? ' ← us' : '';
-    console.log(`  ${r.score.toFixed(3)}  ${r.name}${marker}`);
-  }
-  const ourRank = ranking.findIndex((r) => r.ours) + 1;
-  console.log(`\n  Our rank: ${ourRank} / ${ranking.length}`);
-
-  const summary = {
-    runAt: new Date().toISOString(),
-    benchmark: 'beir-nfcorpus',
-    queries: N,
-    corpusSize: corpus.length,
-    metrics: { ndcg10, mrr10, recall10, recall100, avgQueryLatencyMs: queryMs / N },
-    baselines: BASELINES_NDCG10,
-    ourRank,
-    leaderboardLength: ranking.length,
-  };
-  mkdirSync(RUNS_DIR, { recursive: true });
-  const stamp = summary.runAt.replace(/[:.]/g, '-');
-  writeFileSync(join(RUNS_DIR, `beir-nfcorpus-${stamp}.json`), JSON.stringify(summary, null, 2));
-  writeFileSync(join(RUNS_DIR, 'beir-nfcorpus-latest.json'), JSON.stringify(summary, null, 2));
-  console.log(`\nWrote ${join(RUNS_DIR, `beir-nfcorpus-${stamp}.json`)}`);
-
-  process.exit(0);
-}
-
-main().catch((err) => { console.error(err); process.exit(1); });

package/scripts/run-beir-rrf-ablation.mjs

@@ -1,221 +0,0 @@
-#!/usr/bin/env node
-// run-beir-rrf-ablation.mjs — RRF weight/k ablation matrix on a single
-// BEIR dataset (ADR-087).
-//
-// Pre-requisite: BGE doc embeddings cached at <dataset>/bge-cache/
-// (run scripts/run-beir-bge.mjs first).
-//
-// Reports:
-//   - Fixed default: k=60, equal weights (the headline number we ship)
-//   - Ablations: k=30/60/120 × weights (0.8/1.0, 1.0/1.0, 1.2/0.8 dense/bm25)
-//   - Bootstrap 95% CI on the fixed-default config only
-//
-// Usage:
-//   cd /tmp/beir-nfcorpus
-//   node /path/to/scripts/run-beir-rrf-ablation.mjs
-//   cd /tmp/beir-scifact
-//   BEIR_DATA_DIR=/tmp/beir-scifact/scifact node /path/to/scripts/run-beir-rrf-ablation.mjs
-
-import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
-import { fileURLToPath } from 'node:url';
-import { dirname, join, resolve } from 'node:path';
-import { performance } from 'node:perf_hooks';
-
-const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
-const CLI_ROOT = resolve(SCRIPT_DIR, '..');
-const RUFLO_ROOT = resolve(SCRIPT_DIR, '../../../..');
-const RUNS_DIR = join(RUFLO_ROOT, 'docs', 'benchmarks', 'runs');
-
-const DATA_DIR = process.env.BEIR_DATA_DIR || '/tmp/beir-nfcorpus/nfcorpus';
-const BGE_MODEL = process.env.BGE_MODEL || 'Xenova/bge-base-en-v1.5';
-const CACHE_DIR = join(dirname(DATA_DIR), 'bge-cache');
-
-const BASELINES = {
-  nfcorpus: { 'BM25': 0.325, 'BGE-large-pub': 0.380, 'SPLADE++': 0.347 },
-  scifact:  { 'BM25': 0.679, 'BGE-large-pub': 0.722, 'SPLADE++': 0.704 },
-};
-function detectDataset(p) { const x = p.toLowerCase(); return Object.keys(BASELINES).find(d => x.includes(d)) || 'nfcorpus'; }
-const DATASET = detectDataset(DATA_DIR);
-
-function dcg(rels, k) { let s = 0; for (let i = 0; i < Math.min(rels.length, k); i++) s += (Math.pow(2, rels[i]) - 1) / Math.log2(i + 2); return s; }
-function ndcg(retrieved, qrels, k) { const rels = retrieved.slice(0, k).map((id) => qrels.get(id) ?? 0); const ideal = [...qrels.values()].sort((a, b) => b - a).slice(0, k); const idcg = dcg(ideal, k); return idcg > 0 ? dcg(rels, k) / idcg : 0; }
-function recall(retrieved, qrels, k) { const tot = [...qrels.values()].filter((v) => v > 0).length; if (tot === 0) return 0; let h = 0; for (let i = 0; i < Math.min(retrieved.length, k); i++) if ((qrels.get(retrieved[i]) ?? 0) > 0) h++; return h / tot; }
-function cosine(a, b) { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s; }
-
-function loadJsonl(p) { return readFileSync(p, 'utf-8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); }
-function loadQrels(p) { const q = new Map(); const lines = readFileSync(p, 'utf-8').split('\n'); for (let i = 1; i < lines.length; i++) { if (!lines[i].trim()) continue; const [qid, did, s] = lines[i].split('\t'); if (!q.has(qid)) q.set(qid, new Map()); q.get(qid).set(did, Number(s)); } return q; }
-
-function cachePath(s) { const safe = BGE_MODEL.replace(/\//g, '_'); return join(CACHE_DIR, `${safe}.${s}`); }
-function loadEmbeddings(dim) {
-  if (!existsSync(cachePath('ids')) || !existsSync(cachePath('f32'))) return null;
-  const ids = readFileSync(cachePath('ids'), 'utf-8').split('\n').filter(Boolean);
-  const raw = readFileSync(cachePath('f32'));
-  const buf = new Float32Array(raw.buffer, raw.byteOffset, raw.length / 4);
-  return { ids, embeds: ids.map((_, i) => buf.slice(i * dim, (i + 1) * dim)) };
-}
-
-// RRF — pure function per the original Cormack/Clarke/Buettcher 2009 paper.
-// `rankings` is an array of [{id, score}, ...] sorted descending by score.
-// `weights` is per-system weight (default 1.0 each).
-function rrfFuse(rankings, k = 60, weights) {
-  const scores = new Map();
-  rankings.forEach((ranking, sIdx) => {
-    const w = weights?.[sIdx] ?? 1;
-    ranking.forEach((doc, rankIdx) => {
-      const rank = rankIdx + 1;
-      scores.set(doc.id, (scores.get(doc.id) ?? 0) + w / (k + rank));
-    });
-  });
-  return [...scores.entries()].map(([id, score]) => ({ id, score })).sort((a, b) => b.score - a.score);
-}
-
-// Mulberry32 PRNG for deterministic bootstrap.
-function mulberry32(a) { return function () { a |= 0; a = (a + 0x6D2B79F5) | 0; let t = a; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; }; }
-function bootstrapCI(scores, iter = 10000, rng = mulberry32(42)) {
-  const n = scores.length; const means = new Float64Array(iter);
-  for (let i = 0; i < iter; i++) { let s = 0; for (let j = 0; j < n; j++) s += scores[Math.floor(rng() * n)]; means[i] = s / n; }
-  const sorted = Array.from(means).sort((a, b) => a - b);
-  const lo = sorted[Math.floor(0.025 * iter)]; const hi = sorted[Math.floor(0.975 * iter)];
-  return { mean: scores.reduce((s, v) => s + v, 0) / n, lo, hi };
-}
-
-async function main() {
-  console.log(`# RRF ablation on BEIR ${DATASET} (ADR-087)`);
-  console.log(`Data: ${DATA_DIR}\nDense: ${BGE_MODEL}\n`);
-
-  const corpus = loadJsonl(join(DATA_DIR, 'corpus.jsonl'));
-  const queries = loadJsonl(join(DATA_DIR, 'queries.jsonl'));
-  const qrels = loadQrels(join(DATA_DIR, 'qrels/test.tsv'));
-  console.log(`Corpus: ${corpus.length} · queries: ${queries.length} · test qrels: ${qrels.size}`);
-
-  // Dense
-  const bge = await import(join(CLI_ROOT, 'dist/src/memory/bge-embedder.js'));
-  const emb = await bge.getBgeEmbedder(BGE_MODEL);
-  if (!emb) { console.error('BGE failed'); process.exit(1); }
-  const cached = loadEmbeddings(emb.dim());
-  if (!cached) { console.error(`No cached embeddings at ${cachePath('f32')}. Run run-beir-bge.mjs first.`); process.exit(2); }
-  const { ids: docIds, embeds: docEmbeds } = cached;
-
-  // BM25
-  const { tokenize, buildCorpusStats, multiFieldBM25 } = await import(join(CLI_ROOT, 'dist/src/memory/hybrid-retrieval.js'));
-  const titleDocs = corpus.map((d) => tokenize(d.title || ''));
-  const textDocs = corpus.map((d) => tokenize(d.text || ''));
-  const titleStats = buildCorpusStats(titleDocs);
-  const textStats = buildCorpusStats(textDocs);
-  const corpusIdxToId = corpus.map((d) => d._id);
-
-  // Pre-compute per-query rankings for both systems ONCE, so the ablation
-  // matrix below just permutes RRF k/weights, not the underlying retrievals.
-  const queriesById = new Map(queries.map((q) => [q._id, q.text]));
-  const evalQids = [...qrels.keys()];
-
-  console.log(`Pre-computing dense + BM25 rankings for ${evalQids.length} queries...`);
-  const cached_dense = new Map();
-  const cached_bm25 = new Map();
-  const TOP_K_RETRIEVE = 1000;  // enough for stable RRF
-  const tPre = performance.now();
-  for (const qid of evalQids) {
-    const qtext = queriesById.get(qid);
-    if (!qtext) continue;
-    // Dense
-    const qEmb = await emb.embed(qtext);
-    const dense = docIds.map((id, i) => ({ id, score: cosine(qEmb, docEmbeds[i]) })).sort((a, b) => b.score - a.score).slice(0, TOP_K_RETRIEVE);
-    cached_dense.set(qid, dense);
-    // BM25
-    const qTokens = tokenize(qtext);
-    const bm25 = corpus.map((d, i) => ({ id: corpusIdxToId[i], score: multiFieldBM25(qTokens, titleDocs[i], textDocs[i], titleStats, textStats, 1, 1) })).filter((d) => d.score > 0).sort((a, b) => b.score - a.score).slice(0, TOP_K_RETRIEVE);
-    cached_bm25.set(qid, bm25);
-  }
-  console.log(`Pre-compute done in ${((performance.now() - tPre) / 1000).toFixed(0)}s.\n`);
-
-  // ----- ABLATION GRID -----
-  // FIXED-DEFAULT first (the ship claim). Other variants reported as exploratory.
-  const CONFIGS = [
-    // The headline configurations — defaults fixed before viewing results.
-    { name: 'dense only',             rrf: false, weights: null,           kVal: null },
-    { name: 'BM25 only',              rrf: false, weights: null,           kVal: null, bm25Only: true },
-    { name: 'RRF k=60 equal (default)', rrf: true,  weights: [1.0, 1.0],     kVal: 60 },
-    // Exploratory — disclose as tuned variants, not default.
-    { name: 'RRF k=30 equal',         rrf: true,  weights: [1.0, 1.0],     kVal: 30 },
-    { name: 'RRF k=120 equal',        rrf: true,  weights: [1.0, 1.0],     kVal: 120 },
-    { name: 'RRF k=60 dense=1.2,bm25=0.8', rrf: true,  weights: [1.2, 0.8],     kVal: 60 },
-    { name: 'RRF k=60 dense=0.8,bm25=1.2', rrf: true,  weights: [0.8, 1.2],     kVal: 60 },
-  ];
-
-  function scoreConfig(cfg) {
-    let nSum = 0, r10Sum = 0, r100Sum = 0, n = 0;
-    const perQuery = [];
-    for (const qid of evalQids) {
-      const dense = cached_dense.get(qid);
-      const bm25 = cached_bm25.get(qid);
-      if (!dense) continue;
-      let top100;
-      if (!cfg.rrf) {
-        if (cfg.bm25Only) top100 = (bm25 || []).slice(0, 100).map((d) => d.id);
-        else top100 = dense.slice(0, 100).map((d) => d.id);
-      } else {
-        const fused = rrfFuse([dense, bm25 || []], cfg.kVal, cfg.weights);
-        top100 = fused.slice(0, 100).map((d) => d.id);
-      }
-      const qmap = qrels.get(qid);
-      const ndcg10 = ndcg(top100, qmap, 10);
-      nSum += ndcg10;
-      r10Sum += recall(top100, qmap, 10);
-      r100Sum += recall(top100, qmap, 100);
-      n++;
-      perQuery.push({ qid, ndcg10 });
-    }
-    return {
-      name: cfg.name,
-      ndcg10: nSum / n,
-      recall10: r10Sum / n,
-      recall100: r100Sum / n,
-      n,
-      perQuery,
-    };
-  }
-
-  console.log('Config                                | nDCG@10  R@10    R@100   95% CI');
-  console.log('--------------------------------------|---------------------------------');
-  const results = [];
-  let defaultResult = null;
-  for (const cfg of CONFIGS) {
-    const r = scoreConfig(cfg);
-    const ci = bootstrapCI(r.perQuery.map((q) => q.ndcg10));
-    const ciStr = `[${ci.lo.toFixed(3)}, ${ci.hi.toFixed(3)}]`;
-    console.log(`${cfg.name.padEnd(38)} | ${r.ndcg10.toFixed(4)}  ${r.recall10.toFixed(3)}   ${r.recall100.toFixed(3)}   ${ciStr}`);
-    results.push({ ...r, ci, cfg });
-    if (cfg.name.includes('default')) defaultResult = { ...r, ci, cfg };
-  }
-
-  console.log('\n=== Ship claim (fixed default, k=60, equal weights) ===');
-  console.log(`Dataset: ${DATASET}`);
-  console.log(`nDCG@10: ${defaultResult.ndcg10.toFixed(4)}   95% CI ${`[${defaultResult.ci.lo.toFixed(4)}, ${defaultResult.ci.hi.toFixed(4)}]`}`);
-  for (const [name, score] of Object.entries(BASELINES[DATASET])) {
-    const delta = defaultResult.ndcg10 - score;
-    const inCi = score >= defaultResult.ci.lo && score <= defaultResult.ci.hi;
-    const sig = inCi ? 'n.s.' : (delta > 0 ? 'p<0.05 WIN' : 'p<0.05 LOSS');
-    console.log(`  vs ${name.padEnd(15)} Δ=${(delta >= 0 ? '+' : '') + delta.toFixed(4)}   ${sig}`);
-  }
-
-  // Save
-  const summary = {
-    runAt: new Date().toISOString(),
-    benchmark: `beir-${DATASET}-rrf-ablation`,
-    dataset: DATASET,
-    bgeModel: BGE_MODEL,
-    queries: defaultResult.n,
-    corpusSize: corpus.length,
-    fixedDefault: { name: defaultResult.cfg.name, metrics: { ndcg10: defaultResult.ndcg10, recall10: defaultResult.recall10, recall100: defaultResult.recall100 }, ci: defaultResult.ci, perQuery: defaultResult.perQuery },
-    ablation: results.map((r) => ({ name: r.cfg.name, ndcg10: r.ndcg10, recall10: r.recall10, recall100: r.recall100, ci: r.ci })),
-    baselines: BASELINES[DATASET],
-  };
-  mkdirSync(RUNS_DIR, { recursive: true });
-  const stamp = summary.runAt.replace(/[:.]/g, '-');
-  writeFileSync(join(RUNS_DIR, `beir-${DATASET}-rrf-ablation-${stamp}.json`), JSON.stringify(summary, null, 2));
-  writeFileSync(join(RUNS_DIR, `beir-${DATASET}-rrf-ablation-latest.json`), JSON.stringify(summary, null, 2));
-  console.log(`\nWrote ${join(RUNS_DIR, `beir-${DATASET}-rrf-ablation-${stamp}.json`)}`);
-  process.exit(0);
-}
-
-main().catch((err) => { console.error(err); process.exit(1); });