hippocamp 0.2.11 → 0.2.13

package/scripts/eval-qa.cjs

@@ -1,371 +0,0 @@
-#!/usr/bin/env node
-
-const crypto = require("node:crypto");
-const fsSync = require("node:fs");
-const fs = require("node:fs/promises");
-const os = require("node:os");
-const path = require("node:path");
-const memory = require("./hippocamp-memory.cjs");
-
-const DEFAULT_DATA = ".context/longmemeval/longmemeval_s_cleaned.json";
-const DEFAULT_CACHE = ".context/qa-results.jsonl";
-const DEFAULT_SYSTEMS = "hippocamp,oracle";
-const SYSTEMS = ["hippocamp", "hippocamp-k10", "oracle"];
-const DATA_URL =
-  "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_s_cleaned.json";
-
-// Prompts are verbatim from the official LongMemEval repo (xiaowu0162/LongMemEval):
-// src/generation/run_generation.py for the reader, src/evaluation/evaluate_qa.py for the grader.
-const READER_PROMPT =
-  "I will give you several history chats between you and a user. Please answer the question based on the relevant chat history. Answer the question step by step: first extract all the relevant information, and then reason over the information to get the answer.\n\n\nHistory Chats:\n\n{}\n\nCurrent Date: {}\nQuestion: {}\nAnswer (step by step):";
-
-const BASE_CHECK =
-  "I will give you a question, a correct answer, and a response from a model. Please answer yes if the response contains the correct answer. Otherwise, answer no. If the response is equivalent to the correct answer or contains all the intermediate steps to get the correct answer, you should also answer yes. If the response only contains a subset of the information required by the answer, answer no. ";
-const GRADER_PROMPTS = {
-  "single-session-user": `${BASE_CHECK}\n\nQuestion: {}\n\nCorrect Answer: {}\n\nModel Response: {}\n\nIs the model response correct? Answer yes or no only.`,
-  "single-session-assistant": `${BASE_CHECK}\n\nQuestion: {}\n\nCorrect Answer: {}\n\nModel Response: {}\n\nIs the model response correct? Answer yes or no only.`,
-  "multi-session": `${BASE_CHECK}\n\nQuestion: {}\n\nCorrect Answer: {}\n\nModel Response: {}\n\nIs the model response correct? Answer yes or no only.`,
-  "temporal-reasoning": `${BASE_CHECK}In addition, do not penalize off-by-one errors for the number of days. If the question asks for the number of days/weeks/months, etc., and the model makes off-by-one errors (e.g., predicting 19 days when the answer is 18), the model's response is still correct. \n\nQuestion: {}\n\nCorrect Answer: {}\n\nModel Response: {}\n\nIs the model response correct? Answer yes or no only.`,
-  "knowledge-update":
-    "I will give you a question, a correct answer, and a response from a model. Please answer yes if the response contains the correct answer. Otherwise, answer no. If the response contains some previous information along with an updated answer, the response should be considered as correct as long as the updated answer is the required answer.\n\nQuestion: {}\n\nCorrect Answer: {}\n\nModel Response: {}\n\nIs the model response correct? Answer yes or no only.",
-  "single-session-preference":
-    "I will give you a question, a rubric for desired personalized response, and a response from a model. Please answer yes if the response satisfies the desired response. Otherwise, answer no. The model does not need to reflect all the points in the rubric. The response is correct as long as it recalls and utilizes the user's personal information correctly.\n\nQuestion: {}\n\nRubric: {}\n\nModel Response: {}\n\nIs the model response correct? Answer yes or no only.",
-};
-
-function printHelp() {
-  console.log(`Hippocamp QA eval
-
-LongMemEval-S answer accuracy. A reader model answers each question from the
-retrieved history, and a grader model checks the answer, both with the official
-LongMemEval prompts. Calls go to an OpenAI-compatible chat completions endpoint.
-
-Systems:
-  hippocamp       searchMemory top 5 snippets
-  hippocamp-k10   searchMemory top 10 snippets
-  oracle          the labeled answer sessions, in full
-
-Usage:
-  npm run eval:qa -- [--data PATH] [--limit N] [--systems LIST] [--cache PATH] [--dry-run]
-
-Options:
-  --data PATH      LongMemEval-S JSON. Default: ${DEFAULT_DATA}
-  --limit N        Evaluate N questions spread evenly across the set. Default: all answerable questions
-  --systems LIST   Comma-separated systems. Default: ${DEFAULT_SYSTEMS}
-  --cache PATH     JSONL results cache; reruns only pay for missing calls. Default: ${DEFAULT_CACHE}
-  --concurrency N  Parallel reader and grader calls. Default: 6
-  --dry-run        Print jobs and prompt token estimates without any API call
-  --help           Show this help
-
-Environment:
-  MANIFEST_BASE_URL, MANIFEST_API_KEY   OpenAI-compatible endpoint and key
-  QA_READER_MODEL                       Default: anthropic/claude-sonnet-5
-  QA_GRADER_MODEL                       Default: openrouter/openai/gpt-4o
-
-Download the data once:
-  mkdir -p .context/longmemeval
-  curl -L -o ${DEFAULT_DATA} \\
-    ${DATA_URL}
-`);
-}
-
-function parsePositiveInteger(value, name) {
-  const number = Number(value);
-
-  if (!Number.isInteger(number) || number < 1) {
-    throw new Error(`${name} must be a positive integer.`);
-  }
-
-  return number;
-}
-
-function parseArgs(argv) {
-  const options = {
-    data: DEFAULT_DATA,
-    limit: null,
-    systems: DEFAULT_SYSTEMS.split(","),
-    cache: DEFAULT_CACHE,
-    concurrency: 6,
-    dryRun: false,
-    help: false,
-  };
-
-  for (let index = 0; index < argv.length; index += 1) {
-    const token = argv[index];
-
-    if (token === "--help" || token === "-h") {
-      options.help = true;
-    } else if (token === "--data") {
-      options.data = argv[++index];
-    } else if (token === "--limit") {
-      options.limit = parsePositiveInteger(argv[++index], "--limit");
-    } else if (token === "--systems") {
-      options.systems = String(argv[++index] || "").split(",").filter(Boolean);
-
-      for (const system of options.systems) {
-        if (!SYSTEMS.includes(system)) {
-          throw new Error(`Unknown system: ${system}. Use ${SYSTEMS.join(", ")}.`);
-        }
-      }
-    } else if (token === "--cache") {
-      options.cache = argv[++index];
-    } else if (token === "--concurrency") {
-      options.concurrency = parsePositiveInteger(argv[++index], "--concurrency");
-    } else if (token === "--dry-run") {
-      options.dryRun = true;
-    } else {
-      throw new Error(`Unknown argument: ${token}`);
-    }
-  }
-
-  return options;
-}
-
-function fill(template, ...values) {
-  return values.reduce((text, value) => text.replace("{}", () => String(value)), template);
-}
-
-function formatSession(sessionId, date, turns) {
-  const lines = [`# Session ${sessionId}`, "", `Date: ${date || "unknown"}`, ""];
-
-  for (const turn of turns || []) {
-    lines.push(`## ${turn.role === "assistant" ? "Assistant" : "User"}`, "", String(turn.content || "").trim(), "");
-  }
-
-  return `${lines.join("\n").trim()}\n`;
-}
-
-function historyString(chunks) {
-  return chunks
-    .sort((left, right) => left.date.localeCompare(right.date))
-    .map((chunk, index) => `\n### Session ${index + 1}:\nSession Date: ${chunk.date}\nSession Content:\n${chunk.content}\n`)
-    .join("");
-}
-
-async function hippocampChunks(item, projectRoot, maxResults) {
-  const pathToIndex = new Map();
-
-  for (let index = 0; index < item.haystack_sessions.length; index += 1) {
-    const relativePath = `sessions/${String(index).padStart(4, "0")}.md`;
-    const content = formatSession(item.haystack_session_ids[index], item.haystack_dates[index], item.haystack_sessions[index]);
-
-    await memory.writeMemoryFile({ scope: "project", path: relativePath, content, projectRoot, sync: false });
-    pathToIndex.set(relativePath, index);
-  }
-
-  const search = await memory.searchMemory({ query: item.question, scope: "project", projectRoot, maxResults });
-
-  return search.results.map((result) => ({
-    date: item.haystack_dates[pathToIndex.get(result.path)],
-    content: `\n${result.snippet}`,
-  }));
-}
-
-function oracleChunks(item) {
-  const answerIds = new Set(item.answer_session_ids.map(String));
-
-  return item.haystack_sessions
-    .map((turns, index) => ({ id: String(item.haystack_session_ids[index]), date: item.haystack_dates[index], turns }))
-    .filter((session) => answerIds.has(session.id))
-    .map((session) => ({
-      date: session.date,
-      content: `\n${JSON.stringify(session.turns.map(({ role, content }) => ({ role, content })))}`,
-    }));
-}
-
-// Models that rejected temperature (Claude 5 does); the official runs use 0 wherever it is accepted.
-const NO_TEMPERATURE = new Set();
-
-async function chat(endpoint, model, content, maxTokens) {
-  for (let attempt = 1; ; attempt += 1) {
-    try {
-      const body = { model, messages: [{ role: "user", content }], max_tokens: maxTokens };
-
-      if (!NO_TEMPERATURE.has(model)) {
-        body.temperature = 0;
-      }
-
-      const response = await fetch(endpoint, {
-        method: "POST",
-        headers: { Authorization: `Bearer ${process.env.MANIFEST_API_KEY}`, "Content-Type": "application/json" },
-        body: JSON.stringify(body),
-      });
-      const text = await response.text();
-
-      // Checks this request, not the set: parallel calls can all be refused before the first one marks the model.
-      if (response.status === 400 && /temperature/i.test(text) && "temperature" in body) {
-        NO_TEMPERATURE.add(model);
-        attempt -= 1;
-        continue;
-      }
-
-      if (!response.ok) {
-        throw Object.assign(new Error(`HTTP ${response.status}: ${text.slice(0, 300)}`), { status: response.status });
-      }
-
-      return JSON.parse(text).choices[0].message.content || "";
-    } catch (error) {
-      const retryable = !error.status || error.status === 429 || error.status >= 500;
-
-      if (!retryable || attempt >= 5) {
-        throw error;
-      }
-
-      await new Promise((resolve) => setTimeout(resolve, 2000 * attempt));
-    }
-  }
-}
-
-function percent(rows) {
-  return `${((rows.filter((row) => row.correct).length / Math.max(rows.length, 1)) * 100).toFixed(1)}%`;
-}
-
-async function run(options) {
-  if (!fsSync.existsSync(options.data)) {
-    throw new Error(`Missing ${options.data}. Run with --help for the download command.`);
-  }
-
-  const reader = process.env.QA_READER_MODEL || "anthropic/claude-sonnet-5";
-  const grader = process.env.QA_GRADER_MODEL || "openrouter/openai/gpt-4o";
-  const baseUrl = String(process.env.MANIFEST_BASE_URL || "").replace(/\/+$/, "");
-  const endpoint = `${baseUrl.endsWith("/v1") ? baseUrl : `${baseUrl}/v1`}/chat/completions`;
-  // The reader prompt's hash is in the key, so a change to the data or to retrieval never reuses a stale answer.
-  const promptHash = (prompt) => crypto.createHash("sha1").update(prompt).digest("hex").slice(0, 12);
-  const key = (system, id, hash) => `${system}|${reader}|${grader}|${id}|${hash}`;
-
-  const answerable = JSON.parse(await fs.readFile(options.data, "utf8")).filter(
-    (item) => !String(item.question_id).endsWith("_abs"),
-  );
-  const limit = Math.min(options.limit || answerable.length, answerable.length);
-  const items = Array.from({ length: limit }, (_, index) => answerable[Math.floor((index * answerable.length) / limit)]);
-  const done = new Map();
-
-  if (fsSync.existsSync(options.cache)) {
-    for (const line of (await fs.readFile(options.cache, "utf8")).split("\n").filter(Boolean)) {
-      const row = JSON.parse(line);
-      done.set(`${row.system}|${row.reader}|${row.grader}|${row.id}|${row.promptHash}`, row);
-    }
-  }
-
-  // Build every prompt first (retrieval is local and sequential), then call models concurrently.
-  const jobs = [];
-  const current = new Map();
-  const sandboxRoot = await fs.mkdtemp(path.join(os.tmpdir(), "hippocamp-eval-qa-"));
-  const previousGlobalRoot = process.env.HIPPOCAMP_GLOBAL_ROOT;
-
-  process.env.HIPPOCAMP_GLOBAL_ROOT = path.join(sandboxRoot, "lagoon");
-
-  try {
-    for (const [position, item] of items.entries()) {
-      const projectRoot = path.join(sandboxRoot, "questions", String(position));
-
-      for (const system of options.systems) {
-        const chunks =
-          system === "oracle" ? oracleChunks(item) : await hippocampChunks(item, projectRoot, system === "hippocamp-k10" ? 10 : 5);
-        const prompt = fill(READER_PROMPT, historyString(chunks), item.question_date, item.question);
-        const hash = promptHash(prompt);
-
-        current.set(`${system}|${item.question_id}`, key(system, item.question_id, hash));
-
-        if (!done.has(key(system, item.question_id, hash))) {
-          jobs.push({ system, item, prompt, hash });
-        }
-      }
-
-      await fs.rm(process.env.HIPPOCAMP_GLOBAL_ROOT, { recursive: true, force: true });
-    }
-  } finally {
-    if (previousGlobalRoot === undefined) {
-      delete process.env.HIPPOCAMP_GLOBAL_ROOT;
-    } else {
-      process.env.HIPPOCAMP_GLOBAL_ROOT = previousGlobalRoot;
-    }
-
-    await fs.rm(sandboxRoot, { recursive: true, force: true });
-  }
-
-  console.error(`${items.length} questions, ${jobs.length} new jobs; reader ${reader}, grader ${grader}`);
-
-  for (const system of options.systems) {
-    const chars = jobs.filter((job) => job.system === system).reduce((sum, job) => sum + job.prompt.length, 0);
-    console.error(`  ${system}: ${Math.round(chars / 4)} reader prompt tokens (estimate)`);
-  }
-
-  if (options.dryRun) {
-    return;
-  }
-
-  if (jobs.length && (!baseUrl || !process.env.MANIFEST_API_KEY)) {
-    throw new Error("Set MANIFEST_BASE_URL and MANIFEST_API_KEY.");
-  }
-
-  await fs.mkdir(path.dirname(path.resolve(options.cache)), { recursive: true });
-  let next = 0;
-  let finished = 0;
-  const worker = async () => {
-    while (next < jobs.length) {
-      const job = jobs[next++];
-      const response = await chat(endpoint, reader, job.prompt, 800);
-      const verdict = await chat(
-        endpoint,
-        grader,
-        fill(GRADER_PROMPTS[job.item.question_type], job.item.question, job.item.answer, response),
-        10,
-      );
-      const row = {
-        system: job.system,
-        reader,
-        grader,
-        id: job.item.question_id,
-        type: job.item.question_type,
-        // Same check as the official evaluate_qa.py: 'yes' in the lowercased verdict.
-        correct: /yes/i.test(verdict),
-        promptHash: job.hash,
-        contextChars: job.prompt.length,
-        response,
-      };
-
-      await fs.appendFile(options.cache, `${JSON.stringify(row)}\n`);
-      done.set(key(row.system, row.id, row.promptHash), row);
-
-      if (++finished % 20 === 0) {
-        console.error(`progress ${finished}/${jobs.length}`);
-      }
-    }
-  };
-
-  await Promise.all(Array.from({ length: options.concurrency }, worker));
-
-  console.log(`LongMemEval-S QA, ${items.length} answerable questions, reader ${reader}, grader ${grader}\n`);
-
-  for (const system of options.systems) {
-    const rows = items.map((item) => done.get(current.get(`${system}|${item.question_id}`))).filter(Boolean);
-    const contextChars = Math.round(rows.reduce((sum, row) => sum + row.contextChars, 0) / Math.max(rows.length, 1));
-    console.log(`${system.padEnd(14)} QA ${percent(rows)}  (${rows.length} graded, prompt ${contextChars} chars)`);
-  }
-
-  for (const system of options.systems) {
-    const rows = items.map((item) => done.get(current.get(`${system}|${item.question_id}`))).filter(Boolean);
-    console.log(`\n${system}`);
-
-    for (const type of [...new Set(rows.map((row) => row.type))].sort()) {
-      const byType = rows.filter((row) => row.type === type);
-      console.log(`  ${type.padEnd(26)} ${percent(byType).padStart(6)}  (${byType.length})`);
-    }
-  }
-}
-
-async function main() {
-  const options = parseArgs(process.argv.slice(2));
-
-  if (options.help) {
-    printHelp();
-    return;
-  }
-
-  await run(options);
-}
-
-main().catch((error) => {
-  console.error(error.message);
-  process.exit(1);
-});

package/scripts/eval-retrieval.cjs

@@ -1,323 +0,0 @@
-#!/usr/bin/env node
-
-const fsSync = require("node:fs");
-const fs = require("node:fs/promises");
-const os = require("node:os");
-const path = require("node:path");
-const memory = require("./hippocamp-memory.cjs");
-
-const MAX_TOP_K = 20;
-const DEFAULT_DATA = ".context/longmemeval/longmemeval_s_cleaned.json";
-const DATA_URL =
-  "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_s_cleaned.json";
-
-function printHelp() {
-  console.log(`Hippocamp retrieval eval
-
-Retrieval-only LongMemEval-S check. No model or API key is used.
-Each question's sessions are written as memory files in a temporary Lagoon,
-then searchMemory and an in-process BM25 reference retrieve the top K.
-
-Usage:
-  npm run eval:retrieval -- [--data PATH] [--limit N] [--top-k K]
-
-Options:
-  --data PATH   LongMemEval-S JSON. Default: ${DEFAULT_DATA}
-  --limit N     Evaluate N questions spread evenly across the set. Default: all answerable questions
-  --top-k K     Results retrieved per question, at most ${MAX_TOP_K} (the search_memory cap). Default: 5
-  --help        Show this help
-
-Download the data once:
-  mkdir -p .context/longmemeval
-  curl -L -o ${DEFAULT_DATA} \\
-    ${DATA_URL}
-`);
-}
-
-function parsePositiveInteger(value, name) {
-  const number = Number(value);
-
-  if (!Number.isInteger(number) || number < 1) {
-    throw new Error(`${name} must be a positive integer.`);
-  }
-
-  return number;
-}
-
-function parseArgs(argv) {
-  const options = { data: DEFAULT_DATA, limit: null, topK: 5, help: false };
-
-  for (let index = 0; index < argv.length; index += 1) {
-    const token = argv[index];
-
-    if (token === "--help" || token === "-h") {
-      options.help = true;
-    } else if (token === "--data") {
-      options.data = argv[++index];
-    } else if (token === "--limit") {
-      options.limit = parsePositiveInteger(argv[++index], "--limit");
-    } else if (token === "--top-k") {
-      options.topK = parsePositiveInteger(argv[++index], "--top-k");
-
-      if (options.topK > MAX_TOP_K) {
-        throw new Error(`--top-k must be at most ${MAX_TOP_K}, the search_memory result cap.`);
-      }
-    } else {
-      throw new Error(`Unknown argument: ${token}`);
-    }
-  }
-
-  return options;
-}
-
-function formatSession(sessionId, date, turns) {
-  const lines = [`# Session ${sessionId}`, "", `Date: ${date || "unknown"}`, ""];
-
-  for (const turn of turns || []) {
-    lines.push(`## ${turn.role === "assistant" ? "Assistant" : "User"}`, "", String(turn.content || "").trim(), "");
-  }
-
-  return `${lines.join("\n").trim()}\n`;
-}
-
-function tokenize(value) {
-  return (
-    String(value || "")
-      .normalize("NFKD")
-      .replace(/[̀-ͯ]/g, "")
-      .toLowerCase()
-      .match(/[a-z0-9]+/g) || []
-  );
-}
-
-function rankBm25(query, documents) {
-  const tokenized = documents.map((document) => tokenize(document.content));
-  const averageLength = tokenized.reduce((sum, tokens) => sum + tokens.length, 0) / Math.max(tokenized.length, 1);
-  const documentFrequency = new Map();
-
-  for (const tokens of tokenized) {
-    for (const token of new Set(tokens)) {
-      documentFrequency.set(token, (documentFrequency.get(token) || 0) + 1);
-    }
-  }
-
-  const queryTokens = [...new Set(tokenize(query))];
-
-  return documents
-    .map((document, index) => {
-      const frequencies = new Map();
-
-      for (const token of tokenized[index]) {
-        frequencies.set(token, (frequencies.get(token) || 0) + 1);
-      }
-
-      let score = 0;
-
-      for (const token of queryTokens) {
-        const frequency = frequencies.get(token) || 0;
-
-        if (!frequency) {
-          continue;
-        }
-
-        const matching = documentFrequency.get(token);
-        const idf = Math.log(1 + (documents.length - matching + 0.5) / (matching + 0.5));
-        const lengthRatio = tokenized[index].length / Math.max(averageLength, 1);
-        score += idf * ((frequency * 2.5) / (frequency + 1.5 * (0.25 + 0.75 * lengthRatio)));
-      }
-
-      return { document, score };
-    })
-    .sort((left, right) => right.score - left.score || left.document.index - right.document.index)
-    .map((item) => item.document);
-}
-
-// Words too common to show that a question and its evidence share wording.
-const STOPWORDS = new Set(
-  "about after also been before could does from have into just many much over some such than that their them then there these they this what when where which while with would your".split(" "),
-);
-
-function contentTokens(value) {
-  return new Set(tokenize(value).filter((token) => token.length > 3 && !STOPWORDS.has(token)));
-}
-
-function squashSpace(value) {
-  return String(value || "").replace(/\s+/g, " ").trim();
-}
-
-function retrievalMetrics(retrievedIds, answerIds) {
-  const expected = new Set(answerIds);
-  const hits = retrievedIds.filter((id) => expected.has(id)).length;
-  const firstRank = retrievedIds.findIndex((id) => expected.has(id));
-
-  return {
-    recall: hits / expected.size,
-    hit: hits ? 1 : 0,
-    mrr: firstRank === -1 ? 0 : 1 / (firstRank + 1),
-  };
-}
-
-function addMetrics(bucket, system, metrics) {
-  const total = (bucket[system] ||= { recall: 0, hit: 0, mrr: 0, evidence: 0, contextChars: 0, latencyMs: 0, count: 0 });
-
-  for (const key of ["recall", "hit", "mrr", "evidence", "contextChars", "latencyMs"]) {
-    total[key] += metrics[key] ?? Number.NaN;
-  }
-
-  total.count += 1;
-}
-
-function formatRow(label, total) {
-  const average = (key) => total[key] / total.count;
-
-  return [
-    label.padEnd(12),
-    `Recall@K ${(average("recall") * 100).toFixed(1)}%`,
-    `Hit@K ${(average("hit") * 100).toFixed(1)}%`,
-    `MRR ${average("mrr").toFixed(3)}`,
-    Number.isNaN(total.evidence) ? "evidence  n/a " : `evidence ${(average("evidence") * 100).toFixed(1)}%`,
-    `context ${Math.round(average("contextChars"))} chars`,
-    `latency ${average("latencyMs").toFixed(0)} ms`,
-  ].join("  ");
-}
-
-async function run(options) {
-  if (!fsSync.existsSync(options.data)) {
-    throw new Error(`Missing ${options.data}. Run with --help for the download command.`);
-  }
-
-  const answerable = JSON.parse(await fs.readFile(options.data, "utf8")).filter(
-    (item) => !String(item.question_id).endsWith("_abs"),
-  );
-  const limit = Math.min(options.limit || answerable.length, answerable.length);
-  const items = Array.from({ length: limit }, (_, index) => answerable[Math.floor((index * answerable.length) / limit)]);
-  const sandboxRoot = await fs.mkdtemp(path.join(os.tmpdir(), "hippocamp-eval-retrieval-"));
-  const previousGlobalRoot = process.env.HIPPOCAMP_GLOBAL_ROOT;
-  const overall = {};
-  const byType = {};
-  const misses = [];
-  let withEvidenceTurns = 0;
-
-  process.env.HIPPOCAMP_GLOBAL_ROOT = path.join(sandboxRoot, "lagoon");
-
-  try {
-    for (const [position, item] of items.entries()) {
-      const projectRoot = path.join(sandboxRoot, "questions", String(position));
-      const documents = [];
-      const pathToSessionId = new Map();
-
-      for (let index = 0; index < item.haystack_sessions.length; index += 1) {
-        const sessionId = String(item.haystack_session_ids[index]);
-        const relativePath = `sessions/${String(index).padStart(4, "0")}.md`;
-        const content = formatSession(sessionId, item.haystack_dates[index], item.haystack_sessions[index]);
-
-        await memory.writeMemoryFile({ scope: "project", path: relativePath, content, projectRoot, sync: false });
-        documents.push({ index, content, sessionId });
-        pathToSessionId.set(relativePath, sessionId);
-      }
-
-      const answerIds = item.answer_session_ids.map(String);
-      const evidenceTurns = item.haystack_sessions
-        .filter((_, index) => answerIds.includes(String(item.haystack_session_ids[index])))
-        .flat()
-        .filter((turn) => turn.has_answer)
-        .map((turn) => squashSpace(turn.content));
-      withEvidenceTurns += evidenceTurns.length ? 1 : 0;
-      let startedAt = performance.now();
-      const search = await memory.searchMemory({
-        query: item.question,
-        scope: "project",
-        projectRoot,
-        maxResults: options.topK,
-      });
-      const hippocamp = {
-        ...retrievalMetrics(search.results.map((result) => pathToSessionId.get(result.path)), answerIds),
-        latencyMs: performance.now() - startedAt,
-        contextChars: search.results.reduce((sum, result) => sum + result.snippet.length, 0),
-        // The snippet itself comes from a turn marked has_answer, not only the right session.
-        evidence: search.results.some((result) =>
-          result.snippet
-            .split(/\n\s*\n/)
-            .map(squashSpace)
-            .some((part) => evidenceTurns.some((turn) => turn.includes(part) || part.includes(turn))),
-        )
-          ? 1
-          : 0,
-      };
-
-      if (!hippocamp.hit && evidenceTurns.length) {
-        const shared = [...contentTokens(item.question)].filter((token) =>
-          evidenceTurns.some((turn) => contentTokens(turn).has(token)),
-        );
-        misses.push({ type: item.question_type, sharedWords: shared.length });
-      }
-
-      startedAt = performance.now();
-      const bm25Documents = rankBm25(item.question, documents).slice(0, options.topK);
-      const bm25 = {
-        ...retrievalMetrics(bm25Documents.map((document) => document.sessionId), answerIds),
-        latencyMs: performance.now() - startedAt,
-        contextChars: bm25Documents.reduce((sum, document) => sum + document.content.length, 0),
-        evidence: null,
-      };
-
-      for (const bucket of [overall, (byType[item.question_type] ||= {})]) {
-        addMetrics(bucket, "hippocamp", hippocamp);
-        addMetrics(bucket, "bm25", bm25);
-      }
-
-      await fs.rm(process.env.HIPPOCAMP_GLOBAL_ROOT, { recursive: true, force: true });
-
-      if ((position + 1) % 50 === 0) {
-        console.error(`progress ${position + 1}/${items.length}`);
-      }
-    }
-  } finally {
-    if (previousGlobalRoot === undefined) {
-      delete process.env.HIPPOCAMP_GLOBAL_ROOT;
-    } else {
-      process.env.HIPPOCAMP_GLOBAL_ROOT = previousGlobalRoot;
-    }
-
-    await fs.rm(sandboxRoot, { recursive: true, force: true });
-  }
-
-  console.log(`LongMemEval-S retrieval, ${items.length} answerable questions, K=${options.topK}`);
-  console.log("bm25 returns whole sessions; hippocamp returns bounded snippets.\n");
-
-  for (const [system, total] of Object.entries(overall)) {
-    console.log(formatRow(system, total));
-  }
-
-  const noSharedWords = misses.filter((miss) => miss.sharedWords === 0).length;
-  console.log(
-    `\nevidence: a hippocamp snippet is text from a has_answer turn (${withEvidenceTurns} questions have such turns).`,
-  );
-  console.log(
-    `wording: ${misses.length} hippocamp misses (Hit@K 0); ${noSharedWords} share no content word with the answer turns.`,
-  );
-
-  for (const [type, systems] of Object.entries(byType)) {
-    console.log(`\n${type} (${systems.bm25.count})`);
-
-    for (const [system, total] of Object.entries(systems)) {
-      console.log(`  ${formatRow(system, total)}`);
-    }
-  }
-}
-
-async function main() {
-  const options = parseArgs(process.argv.slice(2));
-
-  if (options.help) {
-    printHelp();
-    return;
-  }
-
-  await run(options);
-}
-
-main().catch((error) => {
-  console.error(error.message);
-  process.exit(1);
-});