nlm-memory 0.4.2 → 0.5.1

package/scripts/extract-triples.mjs

@@ -1,207 +0,0 @@
-#!/usr/bin/env node
-/**
- * Extract (query, surfaced_id, surfaced_body, label, weight, source) triples
- * from the NLM telemetry logs for reranker training data.
- *
- * Data sources:
- *   ~/.nlm/hook-log.jsonl  — one row per UserPromptSubmit/SessionStart fire,
- *                            fields: ts, conversationId, promptPreview, wouldInject[]
- *   ~/.nlm/citation-log.jsonl — one row per detected citation,
- *                            fields: ts, conversation_id, cited_id, kind
- *   ~/.nlm/canonical.sqlite — sessions table for surfaced_body lookup
- *
- * Algorithm:
- *   1. Index citations by (conversationId, citedId, kind).
- *   2. Identify "gold conversations": conversations that produced at least one
- *      tool_use citation. These are the only conversations with confirmed
- *      positive signal — surfaced-but-not-cited sessions in them are genuine
- *      hard negatives.
- *   3. For each hook-log entry with wouldInject.length > 0:
- *      - If the conversation is a gold conversation:
- *        * tool_use cited sessions → weight 1.0, source "tool_use" (gold positive)
- *        * NOT cited sessions → weight 0.0, source "hard_negative"
- *      - Prose-only conversations are skipped (signal too noisy).
- *   4. Fetch surfaced_body from SQLite for each (query, surfaced_id) pair.
- *   5. Write JSONL to --output path (or stdout).
- *
- * Dedup: one triple per (query, surfaced_id, source) key — duplicate hook
- * fires for the same (conversationId, id) pair are collapsed.
- *
- * Usage:
- *   node scripts/extract-triples.mjs                    # stdout, last 30d
- *   node scripts/extract-triples.mjs --days 7           # last 7 days
- *   node scripts/extract-triples.mjs --output triples.jsonl
- *   node scripts/extract-triples.mjs --stats            # summary only, no output
- */
-
-import { createReadStream, existsSync, writeFileSync } from "node:fs";
-import { createInterface } from "node:readline";
-import { homedir } from "node:os";
-import { join } from "node:path";
-import { createWriteStream } from "node:fs";
-import Database from "better-sqlite3";
-
-// ── CLI args ─────────────────────────────────────────────────────────────────
-
-const args = process.argv.slice(2);
-const days = (() => {
-  const idx = args.indexOf("--days");
-  return idx !== -1 ? Number.parseInt(args[idx + 1], 10) || 30 : 30;
-})();
-const outputPath = (() => {
-  const idx = args.indexOf("--output");
-  return idx !== -1 ? args[idx + 1] : null;
-})();
-const statsOnly = args.includes("--stats");
-
-const hookLogPath = process.env["NLM_HOOK_LOG"] ?? join(homedir(), ".nlm", "hook-log.jsonl");
-const citationLogPath = process.env["NLM_CITATION_LOG"] ?? join(homedir(), ".nlm", "citation-log.jsonl");
-const dbPath = process.env["NLM_DB_PATH"] ?? join(homedir(), ".nlm", "canonical.sqlite");
-
-const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
-
-// ── Helpers ───────────────────────────────────────────────────────────────────
-
-async function readJsonl(path) {
-  if (!existsSync(path)) return [];
-  const rows = [];
-  const rl = createInterface({ input: createReadStream(path), crlfDelay: Infinity });
-  for await (const line of rl) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-    try {
-      rows.push(JSON.parse(trimmed));
-    } catch {
-      // skip malformed lines
-    }
-  }
-  return rows;
-}
-
-function tsMs(entry, field = "ts") {
-  const v = entry[field];
-  if (typeof v !== "string") return 0;
-  return Date.parse(v);
-}
-
-// ── Step 1: Load citations ────────────────────────────────────────────────────
-
-const citationRows = await readJsonl(citationLogPath);
-
-// Map: conversationId → Set of tool_use cited IDs
-const toolUseCitations = new Map();
-// Set of gold conversations (those with ≥1 tool_use citation)
-const goldConversations = new Set();
-
-for (const row of citationRows) {
-  if (row.kind !== "tool_use") continue;
-  const convId = row.conversation_id;
-  const citedId = row.cited_id;
-  if (!convId || !citedId) continue;
-  if (!toolUseCitations.has(convId)) toolUseCitations.set(convId, new Set());
-  toolUseCitations.get(convId).add(citedId);
-  goldConversations.add(convId);
-}
-
-// ── Step 2: Load hook-log entries in window ───────────────────────────────────
-
-const hookRows = await readJsonl(hookLogPath);
-
-// Collect: for each (query, surfaced_id), record (conversationId, weight, source)
-// Dedup key: `${conversationId}::${query}::${surfaced_id}::${source}`
-const seen = new Set();
-const rawTriples = [];
-
-for (const row of hookRows) {
-  // Skip stop-hook entries
-  if (typeof row.kind === "string") continue;
-  const ts = tsMs(row);
-  if (!ts || ts < cutoff) continue;
-  const conversationId = row.conversationId;
-  const query = row.promptPreview;
-  const wouldInject = Array.isArray(row.wouldInject) ? row.wouldInject : [];
-  if (!conversationId || !query || wouldInject.length === 0) continue;
-  // Only process gold conversations
-  if (!goldConversations.has(conversationId)) continue;
-
-  const cited = toolUseCitations.get(conversationId) ?? new Set();
-
-  for (const surfacedId of wouldInject) {
-    if (typeof surfacedId !== "string") continue;
-    const isCited = cited.has(surfacedId);
-    const source = isCited ? "tool_use" : "hard_negative";
-    const weight = isCited ? 1.0 : 0.0;
-    const key = `${conversationId}::${query}::${surfacedId}::${source}`;
-    if (seen.has(key)) continue;
-    seen.add(key);
-    rawTriples.push({ query, surfaced_id: surfacedId, weight, source });
-  }
-}
-
-if (rawTriples.length === 0) {
-  console.error(
-    `extract-triples: no triples found. ` +
-    `Checked ${goldConversations.size} gold conversations in the last ${days}d. ` +
-    `Run nlm useful-scan and ensure tool_use citations exist in citation-log.jsonl.`,
-  );
-  process.exit(0);
-}
-
-// ── Step 3: Fetch surfaced_body from SQLite ────────────────────────────────────
-
-const uniqueIds = [...new Set(rawTriples.map((t) => t.surfaced_id))];
-
-let bodyById = new Map();
-if (existsSync(dbPath)) {
-  try {
-    const db = new Database(dbPath, { readonly: true });
-    const placeholders = uniqueIds.map(() => "?").join(",");
-    const rows = db.prepare(
-      `SELECT id, body FROM sessions WHERE id IN (${placeholders})`,
-    ).all(...uniqueIds);
-    for (const row of rows) {
-      bodyById.set(row.id, typeof row.body === "string" ? row.body : "");
-    }
-    db.close();
-  } catch (err) {
-    console.error(`extract-triples: sqlite lookup failed — ${err.message}. Bodies will be empty.`);
-  }
-} else {
-  console.error(`extract-triples: db not found at ${dbPath}. Bodies will be empty.`);
-}
-
-// ── Step 4: Assemble final triples ────────────────────────────────────────────
-
-const triples = rawTriples.map((t) => ({
-  query: t.query,
-  surfaced_id: t.surfaced_id,
-  surfaced_body: bodyById.get(t.surfaced_id) ?? "",
-  label: t.weight === 1.0 ? "positive" : "negative",
-  weight: t.weight,
-  source: t.source,
-}));
-
-// ── Stats ─────────────────────────────────────────────────────────────────────
-
-const positives = triples.filter((t) => t.weight === 1.0).length;
-const negatives = triples.filter((t) => t.weight === 0.0).length;
-const withBody = triples.filter((t) => t.surfaced_body.length > 0).length;
-
-console.error(
-  `extract-triples: ${triples.length} triples ` +
-  `(${positives} positive, ${negatives} hard-negative, ${withBody}/${triples.length} with body) ` +
-  `from ${goldConversations.size} gold conversations over last ${days}d`,
-);
-
-if (statsOnly) process.exit(0);
-
-// ── Step 5: Write output ──────────────────────────────────────────────────────
-
-const lines = triples.map((t) => JSON.stringify(t)).join("\n") + "\n";
-
-if (outputPath) {
-  writeFileSync(outputPath, lines, "utf8");
-  console.error(`extract-triples: wrote ${triples.length} rows to ${outputPath}`);
-} else {
-  process.stdout.write(lines);
-}

package/scripts/longmemeval/embedding-cache.ts

@@ -1,77 +0,0 @@
-/**
- * SHA256-keyed on-disk embedding cache. The LongMemEval-S haystack has
- * ~24K session bodies (~19K unique); embedding them via local Ollama takes
- * ~30 min the first time. Reruns must be instant — calibrating retrieval
- * parameters means dozens of re-evaluations, and re-embedding each time
- * would burn hours of wall clock for no signal.
- *
- * Backed by a small SQLite at $LONGMEMEVAL_CACHE_DIR/embeddings.sqlite.
- * Key = sha256(kind + ":" + text); value = Float32Array as BLOB.
- */
-
-import Database from "better-sqlite3";
-import type { Database as DB } from "better-sqlite3";
-import { createHash } from "node:crypto";
-import { mkdirSync } from "node:fs";
-import { dirname } from "node:path";
-import type { EmbeddingKind, LLMClient } from "../../src/ports/llm-client.js";
-
-const CREATE_SQL =
-  "CREATE TABLE IF NOT EXISTS embeddings (key TEXT PRIMARY KEY, vector BLOB NOT NULL)";
-
-export interface EmbeddingCacheOptions {
-  readonly dbPath: string;
-  readonly llm: LLMClient;
-}
-
-export class EmbeddingCache {
-  private readonly db: DB;
-  private readonly llm: LLMClient;
-  private readonly getStmt: ReturnType<DB["prepare"]>;
-  private readonly putStmt: ReturnType<DB["prepare"]>;
-
-  constructor(opts: EmbeddingCacheOptions) {
-    mkdirSync(dirname(opts.dbPath), { recursive: true });
-    this.db = new Database(opts.dbPath);
-    this.db.pragma("journal_mode = WAL");
-    this.db.prepare(CREATE_SQL).run();
-    this.getStmt = this.db.prepare(
-      "SELECT vector FROM embeddings WHERE key = @key",
-    );
-    this.putStmt = this.db.prepare(
-      "INSERT OR REPLACE INTO embeddings (key, vector) VALUES (@key, @vector)",
-    );
-    this.llm = opts.llm;
-  }
-
-  async embed(text: string, kind: EmbeddingKind): Promise<Float32Array> {
-    const key = createHash("sha256").update(`${kind}:${text}`).digest("hex");
-    const row = this.getStmt.get({ key }) as { vector: Buffer } | undefined;
-    if (row) {
-      return new Float32Array(
-        row.vector.buffer,
-        row.vector.byteOffset,
-        row.vector.byteLength / Float32Array.BYTES_PER_ELEMENT,
-      );
-    }
-    const result = await this.llm.embed(text, kind);
-    const blob = Buffer.from(
-      result.vector.buffer,
-      result.vector.byteOffset,
-      result.vector.byteLength,
-    );
-    this.putStmt.run({ key, vector: blob });
-    return result.vector;
-  }
-
-  size(): number {
-    const row = this.db.prepare("SELECT COUNT(*) AS n FROM embeddings").get() as {
-      n: number;
-    };
-    return row.n;
-  }
-
-  close(): void {
-    this.db.close();
-  }
-}

package/scripts/longmemeval/fetch-dataset.sh

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Fetch the LongMemEval-S (small) dataset from HuggingFace into a local cache.
-# Idempotent — skips download if the target file already exists with non-zero size.
-#
-# Source: https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned
-# Variant: longmemeval_s_cleaned.json — 500 questions × ~40 haystack sessions.
-
-set -euo pipefail
-
-CACHE_DIR="${LONGMEMEVAL_CACHE_DIR:-$HOME/.cache/longmemeval}"
-VARIANT="${LONGMEMEVAL_VARIANT:-longmemeval_s_cleaned.json}"
-URL="https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/${VARIANT}"
-TARGET="${CACHE_DIR}/${VARIANT}"
-
-mkdir -p "$CACHE_DIR"
-
-if [[ -s "$TARGET" ]]; then
-  echo "longmemeval-fetch: ${TARGET} already present ($(wc -c <"$TARGET") bytes) — skipping."
-  exit 0
-fi
-
-echo "longmemeval-fetch: downloading ${URL}"
-curl -fsSL --retry 3 -o "${TARGET}.tmp" "$URL"
-mv "${TARGET}.tmp" "$TARGET"
-echo "longmemeval-fetch: wrote ${TARGET} ($(wc -c <"$TARGET") bytes)"

package/scripts/longmemeval/run-harness.ts

@@ -1,315 +0,0 @@
-/**
- * LongMemEval-S baseline harness for NLM.
- *
- * Body-only ingest (skip classifier) + local Ollama nomic-embed-text. For
- * each evaluation instance: spin up an in-memory NLM corpus loaded with
- * the haystack sessions, query in each retrieval mode (keyword / semantic
- * / hybrid+RRF), score R@5 plus the session-body companion metric.
- *
- * Pure body-only retrieval — this measures the retrieval *algorithm*, not
- * the full classifier-in-loop NLM pipeline. The number is comparable to
- * agentmemory's published R@5 because both bench bodies-only.
- *
- * Usage:
- *   node dist/scripts/longmemeval/run-harness.js \
- *     --variant longmemeval_s_cleaned.json \
- *     --modes keyword,semantic,hybrid \
- *     --limit 500 \
- *     --report-dir reports/longmemeval
- *
- * Re-runs are fast: embeddings cache in ~/.cache/longmemeval/embeddings.sqlite
- * keyed by sha256(kind + text). First run = ~30 min embedding; subsequent = seconds.
- */
-
-import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
-import { homedir, tmpdir } from "node:os";
-import { dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-import { RecallService } from "../../src/core/recall/recall-service.js";
-import { SqliteSessionStore } from "../../src/core/storage/sqlite-session-store.js";
-import { OllamaClient } from "../../src/llm/ollama-client.js";
-import type {
-  EmbedResult,
-  EmbeddingKind,
-  LLMClient,
-} from "../../src/ports/llm-client.js";
-import type { RecallMode } from "../../src/shared/types.js";
-import { EmbeddingCache } from "./embedding-cache.js";
-import { scoreOne, aggregate, type SingleScore } from "./scorer.js";
-import { turnsToBody, type LongMemEvalInstance } from "./types.js";
-import { chunkSessionText } from "../../src/core/embedding/chunk-body.js";
-
-interface Args {
-  readonly datasetPath: string;
-  readonly modes: ReadonlyArray<RecallMode>;
-  readonly limit: number;
-  readonly k: number;
-  readonly reportDir: string;
-  readonly cacheDir: string;
-  readonly migrationsDir: string;
-}
-
-function parseArgs(argv: ReadonlyArray<string>): Args {
-  const get = (flag: string, fallback?: string): string => {
-    const i = argv.indexOf(flag);
-    if (i < 0) {
-      if (fallback === undefined) throw new Error(`missing required flag: ${flag}`);
-      return fallback;
-    }
-    return argv[i + 1] ?? "";
-  };
-  const cacheDir =
-    process.env["LONGMEMEVAL_CACHE_DIR"] ?? join(homedir(), ".cache", "longmemeval");
-  const variant = get("--variant", "longmemeval_s_cleaned.json");
-  const datasetPath = join(cacheDir, variant);
-  const modes = get("--modes", "keyword,semantic,hybrid")
-    .split(",")
-    .map((m) => m.trim()) as RecallMode[];
-  const limit = Number.parseInt(get("--limit", "500"), 10);
-  const k = Number.parseInt(get("--k", "5"), 10);
-  const reportDir = get("--report-dir", resolve("reports/longmemeval"));
-  const migrationsDir = resolve(__dirname, "../../migrations");
-  return { datasetPath, modes, limit, k, reportDir, cacheDir, migrationsDir };
-}
-
-/** LLMClient wrapper: routes embed() through the on-disk cache. */
-class CachingEmbedder implements LLMClient {
-  constructor(private readonly cache: EmbeddingCache) {}
-  async embed(text: string, kind: EmbeddingKind): Promise<EmbedResult> {
-    const vector = await this.cache.embed(text, kind);
-    return { vector, model: "nomic-embed-text@cached" };
-  }
-  async classify(): Promise<never> {
-    throw new Error("classify not used in LongMemEval body-only harness");
-  }
-}
-
-interface InstanceResult {
-  readonly question_id: string;
-  readonly question_type: string;
-  readonly by_mode: Record<string, SingleScore & { returnedIds: string[] }>;
-  readonly embed_failures: number;
-}
-
-async function runInstance(
-  instance: LongMemEvalInstance,
-  args: Args,
-  cache: EmbeddingCache,
-  embedder: LLMClient,
-): Promise<InstanceResult> {
-  const needsEmbeddings = args.modes.some(
-    (m) => m === "semantic" || m === "hybrid",
-  );
-  const tmpDir = mkdtempSync(join(tmpdir(), "nlm-lmeval-"));
-  const store = new SqliteSessionStore({
-    dbPath: join(tmpDir, "canonical.sqlite"),
-    migrationsDir: args.migrationsDir,
-  });
-  const bodyById = new Map<string, string>();
-  let embedFailures = 0;
-  const seen = new Set<string>();
-  try {
-    for (let i = 0; i < instance.haystack_sessions.length; i++) {
-      const id = instance.haystack_session_ids[i];
-      const date = instance.haystack_dates[i];
-      const turns = instance.haystack_sessions[i];
-      if (!id || !date || !turns) continue;
-      if (seen.has(id)) continue;
-      seen.add(id);
-      const body = turnsToBody(turns);
-      bodyById.set(id, body);
-      store.insertSessionForTest({
-        id,
-        runtime: "longmemeval",
-        runtimeSessionId: id,
-        startedAt: date,
-        endedAt: date,
-        durationMin: 0,
-        label: "",
-        summary: "",
-        body,
-        status: "closed",
-        transcriptKind: "longmemeval-jsonl",
-        transcriptPath: null,
-        entities: [],
-        decisions: [],
-        open: [],
-      });
-      if (needsEmbeddings) {
-        const chunks = chunkSessionText({ body });
-        for (let c = 0; c < chunks.length; c++) {
-          try {
-            const vector = await cache.embed(chunks[c]!, "document");
-            store.insertChunkEmbeddingForTest(id, c, vector);
-          } catch {
-            // Per-chunk embed failure is non-fatal — successfully embedded
-            // chunks still contribute via max-pool. Counter tracks attempts.
-            embedFailures++;
-          }
-        }
-      }
-    }
-
-    const recall = new RecallService({ store, llm: embedder });
-    const byMode: InstanceResult["by_mode"] = {};
-    for (const mode of args.modes) {
-      const result = await recall.search({
-        query: instance.question,
-        mode,
-        limit: args.k,
-      });
-      const returnedIds = result.results.map((r) => r.id);
-      const returnedBodies = returnedIds.map((id) => bodyById.get(id) ?? "");
-      const score = scoreOne({
-        returnedIds,
-        goldIds: instance.answer_session_ids,
-        returnedBodies,
-        answer: instance.answer,
-        k: args.k,
-      });
-      byMode[mode] = { ...score, returnedIds };
-    }
-    return {
-      question_id: instance.question_id,
-      question_type: instance.question_type,
-      by_mode: byMode,
-      embed_failures: embedFailures,
-    };
-  } finally {
-    store.close();
-    rmSync(tmpDir, { recursive: true, force: true });
-  }
-}
-
-async function main(): Promise<void> {
-  const args = parseArgs(process.argv.slice(2));
-  console.log(`longmemeval-harness: loading ${args.datasetPath}`);
-  const raw = readFileSync(args.datasetPath, "utf8");
-  const dataset = JSON.parse(raw) as LongMemEvalInstance[];
-  const slice = dataset.slice(0, args.limit);
-  console.log(
-    `longmemeval-harness: ${slice.length}/${dataset.length} instances, modes=${args.modes.join(",")}, k=${args.k}`,
-  );
-
-  // Warm the embedding cache and the LLM transport.
-  const llm = new OllamaClient({ embedModel: "nomic-embed-text" });
-  const cache = new EmbeddingCache({
-    dbPath: join(args.cacheDir, "embeddings.sqlite"),
-    llm,
-  });
-  const embedder = new CachingEmbedder(cache);
-
-  console.log(`longmemeval-harness: cache contains ${cache.size()} embeddings on entry`);
-
-  const results: InstanceResult[] = [];
-  const t0 = Date.now();
-  for (let i = 0; i < slice.length; i++) {
-    const inst = slice[i];
-    if (!inst) continue;
-    const result = await runInstance(inst, args, cache, embedder);
-    results.push(result);
-    if ((i + 1) % 10 === 0 || i === slice.length - 1) {
-      const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
-      const cached = cache.size();
-      console.log(
-        `  [${i + 1}/${slice.length}] ${elapsed}s elapsed, cache=${cached}`,
-      );
-    }
-  }
-
-  // Aggregate.
-  const aggregated: Record<string, ReturnType<typeof aggregate>> = {};
-  for (const mode of args.modes) {
-    aggregated[mode] = aggregate(
-      results.map((r) => r.by_mode[mode] as SingleScore).filter(Boolean),
-    );
-  }
-
-  // Per-question-type breakdown.
-  const byType: Record<string, Record<string, ReturnType<typeof aggregate>>> = {};
-  const types = new Set(results.map((r) => r.question_type));
-  for (const t of types) {
-    byType[t] = {};
-    for (const mode of args.modes) {
-      const subset = results
-        .filter((r) => r.question_type === t)
-        .map((r) => r.by_mode[mode] as SingleScore)
-        .filter(Boolean);
-      byType[t]![mode] = aggregate(subset);
-    }
-  }
-
-  const stamp = new Date().toISOString().replace(/[:T]/g, "-").slice(0, 19);
-  const outDir = join(args.reportDir, stamp);
-  mkdirSync(outDir, { recursive: true });
-  const json = {
-    dataset: args.datasetPath,
-    n: results.length,
-    k: args.k,
-    modes: args.modes,
-    aggregate: aggregated,
-    by_question_type: byType,
-    results,
-    elapsed_seconds: (Date.now() - t0) / 1000,
-  };
-  writeFileSync(join(outDir, "results.json"), JSON.stringify(json, null, 2));
-  writeFileSync(join(outDir, "summary.md"), renderSummary(json));
-  console.log(`longmemeval-harness: wrote ${outDir}/`);
-  console.log(renderSummary(json));
-
-  cache.close();
-}
-
-function renderSummary(json: {
-  dataset: string;
-  n: number;
-  k: number;
-  modes: ReadonlyArray<RecallMode>;
-  aggregate: Record<string, ReturnType<typeof aggregate>>;
-  by_question_type: Record<string, Record<string, ReturnType<typeof aggregate>>>;
-  elapsed_seconds: number;
-}): string {
-  const lines: string[] = [];
-  lines.push(`# LongMemEval-S — NLM baseline (body-only, n=${json.n}, k=${json.k})`);
-  lines.push("");
-  lines.push(`Dataset: \`${json.dataset}\``);
-  lines.push(`Elapsed: ${json.elapsed_seconds.toFixed(1)}s`);
-  lines.push("");
-  lines.push("## Aggregate");
-  lines.push("");
-  lines.push(`| Mode | R@${json.k} | Session-body hit |`);
-  lines.push(`| --- | --- | --- |`);
-  for (const mode of json.modes) {
-    const a = json.aggregate[mode];
-    if (!a) continue;
-    lines.push(
-      `| ${mode} | ${(a.recallAtK * 100).toFixed(1)}% | ${(a.sessionBodyHitRate * 100).toFixed(1)}% |`,
-    );
-  }
-  lines.push("");
-  lines.push("## By question type");
-  lines.push("");
-  const types = Object.keys(json.by_question_type).sort();
-  lines.push(
-    `| Question type | ${json.modes.map((m) => `${m} R@${json.k}`).join(" | ")} |`,
-  );
-  lines.push(`| --- | ${json.modes.map(() => "---").join(" | ")} |`);
-  for (const t of types) {
-    const row = json.by_question_type[t]!;
-    const cells = json.modes.map((m) => {
-      const a = row[m];
-      return a ? `${(a.recallAtK * 100).toFixed(1)}% (n=${a.n})` : "—";
-    });
-    lines.push(`| ${t} | ${cells.join(" | ")} |`);
-  }
-  lines.push("");
-  return lines.join("\n");
-}
-
-void main().catch((err) => {
-  console.error("longmemeval-harness: fatal", err);
-  process.exit(1);
-});