@ngockhoale/ukit 3.0.8 → 3.0.9

package/scripts/bench/memory-ablation.mjs

@@ -0,0 +1,495 @@
+#!/usr/bin/env node
+// TASK-004 — retrieval ablation variants + --compare runner
+// (SPEC §5 FR-007/FR-008, §8, §10).
+//
+// Variant adapter contract (consumed by memory-bench.mjs --variant-module and
+// by the --compare mode below):
+//   createVariant(name, {recordsPath, projectRoot})
+//     → { name, index(), query(text, {scope, projectId, limit}) → hits[],
+//         invalidate(generation), close() }
+//   name ∈ 'lexical' | 'inverted' | 'sqlite-fts5'
+//   hits: {id, project_id, status, labels[], score}
+//
+// All three variants apply the SAME imported eligible() policy before
+// ranking — correctness parity is measured, not assumed (SPEC §15).
+// sqlite-fts5 probes `node:sqlite` dynamically; when unavailable the variant
+// is {name, supported:false, reason} and --compare records `unsupported`,
+// never silently dropping it.
+//
+// CLI:
+//   node scripts/bench/memory-ablation.mjs --fixture-root <dir> --compare
+//     [--runs <n>] --out <file.json>
+//
+// Exit codes: 0 = compare written, 1 = usage/fixture error, 2 = crash.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  precisionAtK,
+  recallAtK,
+  scopeLeakRate,
+  falseAuthorityRate,
+  staleHitRate,
+  summarizeRuns,
+} from './memory-metrics.mjs';
+import { readRecordStore } from '../../src/core/memory/recordStore.js';
+import { eligible, resolveEffectiveScope } from '../../src/core/memory/policy.js';
+
+export const VARIANT_NAMES = ['lexical', 'inverted', 'sqlite-fts5'];
+
+const TOP_K = 5;
+const SEARCH_LIMIT = 10;
+const DEFAULT_RUNS = 3;
+// Same binding the bench runner uses (SPEC §7 boundProjectId).
+const BENCH_PROJECT_ID = 'proj-alpha';
+
+const USAGE = `Usage: node scripts/bench/memory-ablation.mjs --fixture-root <dir> --compare --out <file.json> [--runs <n>]
+
+Runs every retrieval variant (lexical, inverted, sqlite-fts5) over the corpus
+manifest queries and writes a per-variant correctness+latency report.
+Unavailable variants are recorded as 'unsupported', never dropped.
+
+Options:
+  --fixture-root <dir>  Corpus dir with records.json + manifest.json (required)
+  --compare             Run all variants and emit the comparison report
+  --out <file.json>     Report output path (required)
+  --runs <n>            Latency repetitions per query (default ${DEFAULT_RUNS})
+  --help                Show this help
+
+Exit codes: 0 = report written, 1 = usage/fixture error, 2 = crash.`;
+
+// ---------- shared scoring (mirrors retrieval.js semantics) ----------
+
+const STOPWORDS = new Set([
+  'the', 'a', 'an', 'and', 'or', 'to', 'for', 'of', 'with', 'in', 'on', 'is', 'are',
+  'this', 'that', 'it', 'as', 'by', 'be', 'use', 'using', 'implement', 'fix', 'task',
+  'cần', 'và', 'là', 'cho', 'một', 'những', 'dùng',
+]);
+const RECORD_TYPE_WEIGHTS = Object.freeze({
+  project_rule: 7, derived_fact: 5, procedure: 5, episode: 3,
+});
+const LEGACY_SCOPE_MATCH = {
+  user: (record) => record.scope === 'user',
+  project: (record) => record.scope === 'repo' || record.scope === 'task',
+  session: (record) => record.type === 'episode' || record.scope === 'session',
+};
+
+function normalize(text) {
+  return String(text ?? '')
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s./:_-]/gu, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function tokenize(text) {
+  return normalize(text)
+    .split(/\s+/)
+    .filter((token) => token && !STOPWORDS.has(token) && token.length > 1);
+}
+
+function scoreRecord(record, queryTokens) {
+  const typeWeight = RECORD_TYPE_WEIGHTS[record.type] ?? 1;
+  if (queryTokens.length === 0) {
+    return typeWeight * Math.max(0.1, record.confidence ?? 1);
+  }
+  const recordTokens = new Set(tokenize(`${record.text ?? ''} ${record.provenance ?? ''}`));
+  let matched = 0;
+  for (const token of queryTokens) {
+    if (recordTokens.has(token)) matched += 1;
+  }
+  if (matched === 0) return 0;
+  const recencyBonus = record.created_at > 0 ? Math.min(1, record.created_at / Date.now()) : 0;
+  return typeWeight * matched * Math.max(0.1, record.confidence ?? 1) + recencyBonus;
+}
+
+function recordMatchesRequestedScope(record, scope) {
+  if (scope === 'all') return true;
+  const matcher = LEGACY_SCOPE_MATCH[scope];
+  return matcher ? matcher(record) : (record.scope === scope || record.type === scope);
+}
+
+function toHit(record, score) {
+  const label = record.meta?.bench_label;
+  return {
+    id: record.id,
+    project_id: record.project_id,
+    status: record.status,
+    labels: Array.isArray(label) ? label : (label ? [label] : []),
+    score,
+  };
+}
+
+function rankRecords(records, text, { scope = 'all', projectId = null, limit = SEARCH_LIMIT } = {}) {
+  const effective = resolveEffectiveScope({ projectId }, scope);
+  if (effective.denied) return [];
+  const ctx = { projectId: effective.projectId, includeUser: effective.includeUser };
+  const queryTokens = tokenize(text);
+  return records
+    .filter((record) => eligible(record, ctx).ok)
+    .filter((record) => recordMatchesRequestedScope(record, scope))
+    .map((record) => ({ record, score: scoreRecord(record, queryTokens) }))
+    .filter((entry) => entry.score > 0)
+    .sort((a, b) => b.score - a.score
+      || (b.record.created_at ?? 0) - (a.record.created_at ?? 0)
+      || String(b.record.text ?? '').localeCompare(String(a.record.text ?? '')))
+    .slice(0, limit)
+    .map(({ record, score }) => toHit(record, score));
+}
+
+// ---------- lexical variant (full scan per query) ----------
+
+function createLexical({ recordsPath }) {
+  let records = [];
+  return {
+    name: 'lexical',
+    async index() {
+      const { records: loaded } = await readRecordStore(recordsPath);
+      records = loaded;
+    },
+    async query(text, opts = {}) {
+      return rankRecords(records, text, opts);
+    },
+    async invalidate() {},
+    async close() {},
+  };
+}
+
+// ---------- inverted variant (token → postings map) ----------
+
+function createInverted({ recordsPath }) {
+  let records = [];
+  let postings = new Map();
+  let generation = null;
+
+  async function build() {
+    const { records: loaded, generation: gen } = await readRecordStore(recordsPath);
+    records = loaded;
+    generation = gen;
+    postings = new Map();
+    records.forEach((record, i) => {
+      for (const token of new Set(tokenize(`${record.text ?? ''} ${record.provenance ?? ''}`))) {
+        let list = postings.get(token);
+        if (!list) postings.set(token, (list = []));
+        list.push(i);
+      }
+    });
+  }
+
+  return {
+    name: 'inverted',
+    index: build,
+    async query(text, opts = {}) {
+      const { scope = 'all', projectId = null, limit = SEARCH_LIMIT } = opts;
+      const effective = resolveEffectiveScope({ projectId }, scope);
+      if (effective.denied) return [];
+      const ctx = { projectId: effective.projectId, includeUser: effective.includeUser };
+      const queryTokens = tokenize(text);
+      let candidates;
+      if (queryTokens.length === 0) {
+        candidates = records.map((_, i) => i);
+      } else {
+        const seen = new Set();
+        for (const token of queryTokens) {
+          for (const i of postings.get(token) ?? []) seen.add(i);
+        }
+        candidates = [...seen];
+      }
+      return candidates
+        .map((i) => records[i])
+        .filter((record) => eligible(record, ctx).ok)
+        .filter((record) => recordMatchesRequestedScope(record, scope))
+        .map((record) => ({ record, score: scoreRecord(record, queryTokens) }))
+        .filter((entry) => entry.score > 0)
+        .sort((a, b) => b.score - a.score
+          || (b.record.created_at ?? 0) - (a.record.created_at ?? 0)
+          || String(b.record.text ?? '').localeCompare(String(a.record.text ?? '')))
+        .slice(0, limit)
+        .map(({ record, score }) => toHit(record, score));
+    },
+    async invalidate(newGeneration) {
+      if (newGeneration !== generation) await build();
+    },
+    async close() {},
+  };
+}
+
+// ---------- sqlite-fts5 variant (prototype; FTS is a ranking aid only) ----------
+
+async function defaultImportSqlite() {
+  return import('node:sqlite');
+}
+async function createSqliteFts5({ recordsPath, importSqlite = defaultImportSqlite }) {
+  let sqlite;
+  try {
+    sqlite = await importSqlite();
+  } catch (error) {
+    return {
+      name: 'sqlite-fts5',
+      supported: false,
+      reason: `node:sqlite unavailable: ${error?.message ?? error}`,
+    };
+  }
+  if (typeof sqlite?.DatabaseSync !== 'function') {
+    return {
+      name: 'sqlite-fts5',
+      supported: false,
+      reason: 'node:sqlite loaded but DatabaseSync is missing',
+    };
+  }
+
+  let db = null;
+  let generation = null;
+
+  async function build() {
+    const { records, generation: gen } = await readRecordStore(recordsPath);
+    generation = gen;
+    if (db) { try { db.close(); } catch { /* reopen */ } }
+    db = new sqlite.DatabaseSync(':memory:');
+    db.exec('CREATE TABLE records_mirror (id TEXT PRIMARY KEY, doc TEXT NOT NULL)');
+    db.exec("CREATE VIRTUAL TABLE records_fts USING fts5(text, content='')");
+    const insertMirror = db.prepare('INSERT INTO records_mirror (id, doc) VALUES (?, ?)');
+    const insertFts = db.prepare('INSERT INTO records_fts (rowid, text) VALUES (?, ?)');
+    // FTS indexes eligible text only: context-independent denials (status,
+    // expiry, unverified provenance) are excluded at index time; the full
+    // context-dependent eligible() check still runs before ranking.
+    const indexable = (record) => {
+      if (record == null || typeof record !== 'object') return false;
+      if (record.status !== 'active') return false;
+      if (record.valid_until != null && record.valid_until <= Date.now()) return false;
+      return !['learning-observation', 'learning-candidate', 'delta-overlay']
+        .includes(record.provenance);
+    };
+    let row = 0;
+    for (const record of records) {
+      row += 1;
+      insertMirror.run(record.id, JSON.stringify(record));
+      if (indexable(record)) {
+        insertFts.run(row, `${record.text ?? ''} ${record.provenance ?? ''}`);
+      }
+    }
+  }
+
+  return {
+    name: 'sqlite-fts5',
+    index: build,
+    async query(text, opts = {}) {
+      const { scope = 'all', projectId = null, limit = SEARCH_LIMIT } = opts;
+      const effective = resolveEffectiveScope({ projectId }, scope);
+      if (effective.denied) return [];
+      const ctx = { projectId: effective.projectId, includeUser: effective.includeUser };
+      const queryTokens = tokenize(text);
+
+      let rows;
+      if (queryTokens.length === 0) {
+        rows = db.prepare('SELECT doc FROM records_mirror').all();
+      } else {
+        // Phrase-match each token so FTS tokenizer differences cannot drop a
+        // candidate our tokenizer would keep; OR mirrors token-overlap recall.
+        const match = queryTokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(' OR ');
+        rows = db.prepare(
+          'SELECT m.doc FROM records_fts JOIN records_mirror m ON m.rowid = records_fts.rowid '
+          + 'WHERE records_fts MATCH ?',
+        ).all(match);
+      }
+      return rows
+        .map((row) => JSON.parse(row.doc))
+        .filter((record) => eligible(record, ctx).ok)
+        .filter((record) => recordMatchesRequestedScope(record, scope))
+        .map((record) => ({ record, score: scoreRecord(record, queryTokens) }))
+        .filter((entry) => entry.score > 0)
+        .sort((a, b) => b.score - a.score
+          || (b.record.created_at ?? 0) - (a.record.created_at ?? 0)
+          || String(b.record.text ?? '').localeCompare(String(a.record.text ?? '')))
+        .slice(0, limit)
+        .map(({ record, score }) => toHit(record, score));
+    },
+    async invalidate(newGeneration) {
+      if (newGeneration !== generation) await build();
+    },
+    async close() {
+      if (db) { try { db.close(); } catch { /* already closed */ } db = null; }
+    },
+  };
+}
+
+// ---------- adapter factory ----------
+
+export async function createVariant(name, { recordsPath, projectRoot, importSqlite } = {}) {
+  switch (name) {
+    case 'lexical': return createLexical({ recordsPath, projectRoot });
+    case 'inverted': return createInverted({ recordsPath, projectRoot });
+    case 'sqlite-fts5': return createSqliteFts5({ recordsPath, projectRoot, importSqlite });
+    default: {
+      const err = new Error(`unknown variant: ${name}`);
+      err.unsupported = true;
+      return { name, supported: false, reason: err.message };
+    }
+  }
+}
+
+// ---------- --compare mode ----------
+
+function loadCorpus(fixtureRoot) {
+  const recordsPath = path.join(fixtureRoot, 'records.json');
+  const manifestPath = path.join(fixtureRoot, 'manifest.json');
+  if (!fs.existsSync(recordsPath)) throw new Error(`records.json not found: ${recordsPath}`);
+  if (!fs.existsSync(manifestPath)) throw new Error(`manifest.json not found: ${manifestPath}`);
+  const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+  if (!Array.isArray(manifest.queries)) throw new Error('manifest.json has no queries[] array');
+  return { recordsPath, manifest };
+}
+
+function meanDefined(values) {
+  const nums = values.filter((v) => typeof v === 'number' && Number.isFinite(v));
+  if (nums.length === 0) return 'UNKNOWN';
+  return nums.reduce((a, b) => a + b, 0) / nums.length;
+}
+
+async function measureVariant(name, { recordsPath, projectRoot, queries, manifest, runs, importSqlite }) {
+  const entry = {
+    verdict: 'measured',
+    correctness: {
+      precisionAt5: 'UNKNOWN', recallAt5: 'UNKNOWN', scopeLeakRate: 'UNKNOWN',
+      falseAuthorityRate: 'UNKNOWN', staleHitRate: 'UNKNOWN',
+    },
+    latency: { n: 0, p50: 'UNKNOWN', p95: 'UNKNOWN', p99: 'UNKNOWN' },
+    notes: [],
+  };
+
+  const variant = await createVariant(name, { recordsPath, projectRoot, importSqlite });
+  if (variant.supported === false) {
+    entry.verdict = 'unsupported';
+    entry.reason = variant.reason;
+    entry.notes.push(variant.reason);
+    return entry;
+  }
+
+  try {
+    await variant.index();
+    const allHits = [];
+    const p5 = [];
+    const r5 = [];
+    const durations = [];
+    for (const q of queries) {
+      const scope = q.scope ?? 'all';
+      const projectId = q.boundProjectId ?? BENCH_PROJECT_ID;
+      const hits = await variant.query(q.text, { scope, projectId, limit: SEARCH_LIMIT });
+      allHits.push(...hits);
+      const rankedIds = hits.map((h) => h.id);
+      p5.push(precisionAtK(rankedIds, q.relevantIds, TOP_K));
+      r5.push(recallAtK(rankedIds, q.relevantIds, TOP_K));
+      for (let run = 0; run < runs; run += 1) {
+        const started = performance.now();
+        await variant.query(q.text, { scope, projectId, limit: SEARCH_LIMIT });
+        durations.push(performance.now() - started);
+      }
+    }
+    entry.correctness.precisionAt5 = meanDefined(p5);
+    entry.correctness.recallAt5 = meanDefined(r5);
+    entry.correctness.scopeLeakRate = scopeLeakRate(allHits, BENCH_PROJECT_ID);
+    entry.correctness.falseAuthorityRate = falseAuthorityRate(allHits);
+    entry.correctness.staleHitRate = staleHitRate(allHits, manifest);
+    entry.latency = summarizeRuns(durations);
+  } catch (error) {
+    entry.verdict = 'error';
+    entry.notes.push(String(error?.message ?? error));
+  } finally {
+    try { await variant.close?.(); } catch { /* close errors are not data */ }
+  }
+  return entry;
+}
+
+export async function runCompare({ fixtureRoot, runs = DEFAULT_RUNS, importSqlite } = {}) {
+  const { recordsPath, manifest } = loadCorpus(path.resolve(fixtureRoot));
+  const report = {
+    schemaVersion: 1,
+    kind: 'ablation-compare',
+    corpus: {
+      seed: manifest.seed ?? 'UNKNOWN',
+      tier: manifest.tier ?? 'UNKNOWN',
+      recordCount: manifest.recordCount ?? 'UNKNOWN',
+    },
+    variants: {},
+  };
+  for (const name of VARIANT_NAMES) {
+    report.variants[name] = await measureVariant(name, {
+      recordsPath,
+      projectRoot: path.resolve(fixtureRoot),
+      queries: manifest.queries,
+      manifest,
+      runs,
+      importSqlite,
+    });
+  }
+  return report;
+}
+
+// ---------- CLI ----------
+
+function parseArgs(argv) {
+  const opts = { compare: false, runs: DEFAULT_RUNS };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === '--help' || arg === '-h') return { help: true };
+    const takeValue = () => {
+      i += 1;
+      if (i >= argv.length) throw new Error(`Missing value for ${arg}`);
+      return argv[i];
+    };
+    if (arg === '--fixture-root') opts.fixtureRoot = takeValue();
+    else if (arg === '--out') opts.out = takeValue();
+    else if (arg === '--compare') opts.compare = true;
+    else if (arg === '--runs') opts.runs = Number(takeValue());
+    else throw new Error(`Unknown argument: ${arg}`);
+  }
+  return opts;
+}
+
+async function main() {
+  let opts;
+  try {
+    opts = parseArgs(process.argv.slice(2));
+  } catch (error) {
+    console.error(`[memory-ablation] ${error.message}\n\n${USAGE}`);
+    process.exit(1);
+  }
+  if (opts.help) {
+    console.log(USAGE);
+    process.exit(0);
+  }
+  if (!opts.compare || !opts.fixtureRoot || !opts.out) {
+    console.error(`[memory-ablation] --compare, --fixture-root and --out are required.\n\n${USAGE}`);
+    process.exit(1);
+  }
+  if (!Number.isFinite(opts.runs) || opts.runs < 1) {
+    console.error('[memory-ablation] --runs must be a positive integer.');
+    process.exit(1);
+  }
+  if (!fs.existsSync(path.resolve(opts.fixtureRoot))) {
+    console.error(`[memory-ablation] fixture root not found: ${opts.fixtureRoot}`);
+    process.exit(1);
+  }
+
+  const report = await runCompare({ fixtureRoot: opts.fixtureRoot, runs: opts.runs });
+  for (const [name, v] of Object.entries(report.variants)) {
+    console.log(`[memory-ablation] ${name}: p@5=${v.correctness?.precisionAt5} `
+      + `p95=${v.latency?.p95} verdict=${v.verdict}`);
+  }
+
+  const outPath = path.resolve(opts.out);
+  fs.mkdirSync(path.dirname(outPath), { recursive: true });
+  fs.writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`);
+  console.log(`[memory-ablation] report written: ${outPath}`);
+  process.exit(0);
+}
+
+const invokedAsScript = process.argv[1]
+  && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (invokedAsScript) {
+  main().catch((error) => {
+    console.error(`[memory-ablation] harness crash: ${error?.stack ?? error}`);
+    process.exit(2);
+  });
+}