hippocamp 0.1.1 → 0.2.1

package/scripts/eval-retrieval.cjs

@@ -0,0 +1,276 @@
+#!/usr/bin/env node
+
+const fsSync = require("node:fs");
+const fs = require("node:fs/promises");
+const os = require("node:os");
+const path = require("node:path");
+const memory = require("./hippocamp-memory.cjs");
+
+const MAX_TOP_K = 20;
+const DEFAULT_DATA = ".context/longmemeval/longmemeval_s_cleaned.json";
+const DATA_URL =
+  "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_s_cleaned.json";
+
+function printHelp() {
+  console.log(`Hippocamp retrieval eval
+
+Retrieval-only LongMemEval-S check. No model or API key is used.
+Each question's sessions are written as memory files in a temporary Lagoon,
+then searchMemory and an in-process BM25 reference retrieve the top K.
+
+Usage:
+  npm run eval:retrieval -- [--data PATH] [--limit N] [--top-k K]
+
+Options:
+  --data PATH   LongMemEval-S JSON. Default: ${DEFAULT_DATA}
+  --limit N     Evaluate N questions spread evenly across the set. Default: all answerable questions
+  --top-k K     Results retrieved per question, at most ${MAX_TOP_K} (the search_memory cap). Default: 5
+  --help        Show this help
+
+Download the data once:
+  mkdir -p .context/longmemeval
+  curl -L -o ${DEFAULT_DATA} \\
+    ${DATA_URL}
+`);
+}
+
+function parsePositiveInteger(value, name) {
+  const number = Number(value);
+
+  if (!Number.isInteger(number) || number < 1) {
+    throw new Error(`${name} must be a positive integer.`);
+  }
+
+  return number;
+}
+
+function parseArgs(argv) {
+  const options = { data: DEFAULT_DATA, limit: null, topK: 5, help: false };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const token = argv[index];
+
+    if (token === "--help" || token === "-h") {
+      options.help = true;
+    } else if (token === "--data") {
+      options.data = argv[++index];
+    } else if (token === "--limit") {
+      options.limit = parsePositiveInteger(argv[++index], "--limit");
+    } else if (token === "--top-k") {
+      options.topK = parsePositiveInteger(argv[++index], "--top-k");
+
+      if (options.topK > MAX_TOP_K) {
+        throw new Error(`--top-k must be at most ${MAX_TOP_K}, the search_memory result cap.`);
+      }
+    } else {
+      throw new Error(`Unknown argument: ${token}`);
+    }
+  }
+
+  return options;
+}
+
+function formatSession(sessionId, date, turns) {
+  const lines = [`# Session ${sessionId}`, "", `Date: ${date || "unknown"}`, ""];
+
+  for (const turn of turns || []) {
+    lines.push(`## ${turn.role === "assistant" ? "Assistant" : "User"}`, "", String(turn.content || "").trim(), "");
+  }
+
+  return `${lines.join("\n").trim()}\n`;
+}
+
+function tokenize(value) {
+  return (
+    String(value || "")
+      .normalize("NFKD")
+      .replace(/[̀-ͯ]/g, "")
+      .toLowerCase()
+      .match(/[a-z0-9]+/g) || []
+  );
+}
+
+function rankBm25(query, documents) {
+  const tokenized = documents.map((document) => tokenize(document.content));
+  const averageLength = tokenized.reduce((sum, tokens) => sum + tokens.length, 0) / Math.max(tokenized.length, 1);
+  const documentFrequency = new Map();
+
+  for (const tokens of tokenized) {
+    for (const token of new Set(tokens)) {
+      documentFrequency.set(token, (documentFrequency.get(token) || 0) + 1);
+    }
+  }
+
+  const queryTokens = [...new Set(tokenize(query))];
+
+  return documents
+    .map((document, index) => {
+      const frequencies = new Map();
+
+      for (const token of tokenized[index]) {
+        frequencies.set(token, (frequencies.get(token) || 0) + 1);
+      }
+
+      let score = 0;
+
+      for (const token of queryTokens) {
+        const frequency = frequencies.get(token) || 0;
+
+        if (!frequency) {
+          continue;
+        }
+
+        const matching = documentFrequency.get(token);
+        const idf = Math.log(1 + (documents.length - matching + 0.5) / (matching + 0.5));
+        const lengthRatio = tokenized[index].length / Math.max(averageLength, 1);
+        score += idf * ((frequency * 2.5) / (frequency + 1.5 * (0.25 + 0.75 * lengthRatio)));
+      }
+
+      return { document, score };
+    })
+    .sort((left, right) => right.score - left.score || left.document.index - right.document.index)
+    .map((item) => item.document);
+}
+
+function retrievalMetrics(retrievedIds, answerIds) {
+  const expected = new Set(answerIds);
+  const hits = retrievedIds.filter((id) => expected.has(id)).length;
+  const firstRank = retrievedIds.findIndex((id) => expected.has(id));
+
+  return {
+    recall: hits / expected.size,
+    hit: hits ? 1 : 0,
+    mrr: firstRank === -1 ? 0 : 1 / (firstRank + 1),
+  };
+}
+
+function addMetrics(bucket, system, metrics) {
+  const total = (bucket[system] ||= { recall: 0, hit: 0, mrr: 0, contextChars: 0, latencyMs: 0, count: 0 });
+
+  for (const key of ["recall", "hit", "mrr", "contextChars", "latencyMs"]) {
+    total[key] += metrics[key];
+  }
+
+  total.count += 1;
+}
+
+function formatRow(label, total) {
+  const average = (key) => total[key] / total.count;
+
+  return [
+    label.padEnd(12),
+    `Recall@K ${(average("recall") * 100).toFixed(1)}%`,
+    `Hit@K ${(average("hit") * 100).toFixed(1)}%`,
+    `MRR ${average("mrr").toFixed(3)}`,
+    `context ${Math.round(average("contextChars"))} chars`,
+    `latency ${average("latencyMs").toFixed(0)} ms`,
+  ].join("  ");
+}
+
+async function run(options) {
+  if (!fsSync.existsSync(options.data)) {
+    throw new Error(`Missing ${options.data}. Run with --help for the download command.`);
+  }
+
+  const answerable = JSON.parse(await fs.readFile(options.data, "utf8")).filter(
+    (item) => !String(item.question_id).endsWith("_abs"),
+  );
+  const limit = Math.min(options.limit || answerable.length, answerable.length);
+  const items = Array.from({ length: limit }, (_, index) => answerable[Math.floor((index * answerable.length) / limit)]);
+  const sandboxRoot = await fs.mkdtemp(path.join(os.tmpdir(), "hippocamp-eval-retrieval-"));
+  const previousGlobalRoot = process.env.HIPPOCAMP_GLOBAL_ROOT;
+  const overall = {};
+  const byType = {};
+
+  process.env.HIPPOCAMP_GLOBAL_ROOT = path.join(sandboxRoot, "lagoon");
+
+  try {
+    for (const [position, item] of items.entries()) {
+      const projectRoot = path.join(sandboxRoot, "questions", String(position));
+      const documents = [];
+      const pathToSessionId = new Map();
+
+      for (let index = 0; index < item.haystack_sessions.length; index += 1) {
+        const sessionId = String(item.haystack_session_ids[index]);
+        const relativePath = `sessions/${String(index).padStart(4, "0")}.md`;
+        const content = formatSession(sessionId, item.haystack_dates[index], item.haystack_sessions[index]);
+
+        await memory.writeMemoryFile({ scope: "project", path: relativePath, content, projectRoot, sync: false });
+        documents.push({ index, content, sessionId });
+        pathToSessionId.set(relativePath, sessionId);
+      }
+
+      const answerIds = item.answer_session_ids.map(String);
+      let startedAt = performance.now();
+      const search = await memory.searchMemory({
+        query: item.question,
+        scope: "project",
+        projectRoot,
+        maxResults: options.topK,
+      });
+      const hippocamp = {
+        ...retrievalMetrics(search.results.map((result) => pathToSessionId.get(result.path)), answerIds),
+        latencyMs: performance.now() - startedAt,
+        contextChars: search.results.reduce((sum, result) => sum + result.snippet.length, 0),
+      };
+
+      startedAt = performance.now();
+      const bm25Documents = rankBm25(item.question, documents).slice(0, options.topK);
+      const bm25 = {
+        ...retrievalMetrics(bm25Documents.map((document) => document.sessionId), answerIds),
+        latencyMs: performance.now() - startedAt,
+        contextChars: bm25Documents.reduce((sum, document) => sum + document.content.length, 0),
+      };
+
+      for (const bucket of [overall, (byType[item.question_type] ||= {})]) {
+        addMetrics(bucket, "hippocamp", hippocamp);
+        addMetrics(bucket, "bm25", bm25);
+      }
+
+      await fs.rm(process.env.HIPPOCAMP_GLOBAL_ROOT, { recursive: true, force: true });
+
+      if ((position + 1) % 50 === 0) {
+        console.error(`progress ${position + 1}/${items.length}`);
+      }
+    }
+  } finally {
+    if (previousGlobalRoot === undefined) {
+      delete process.env.HIPPOCAMP_GLOBAL_ROOT;
+    } else {
+      process.env.HIPPOCAMP_GLOBAL_ROOT = previousGlobalRoot;
+    }
+
+    await fs.rm(sandboxRoot, { recursive: true, force: true });
+  }
+
+  console.log(`LongMemEval-S retrieval, ${items.length} answerable questions, K=${options.topK}`);
+  console.log("bm25 returns whole sessions; hippocamp returns bounded snippets.\n");
+
+  for (const [system, total] of Object.entries(overall)) {
+    console.log(formatRow(system, total));
+  }
+
+  for (const [type, systems] of Object.entries(byType)) {
+    console.log(`\n${type} (${systems.bm25.count})`);
+
+    for (const [system, total] of Object.entries(systems)) {
+      console.log(`  ${formatRow(system, total)}`);
+    }
+  }
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+
+  if (options.help) {
+    printHelp();
+    return;
+  }
+
+  await run(options);
+}
+
+main().catch((error) => {
+  console.error(error.message);
+  process.exit(1);
+});

package/scripts/hippocamp-dream.cjs

@@ -10,6 +10,17 @@ const DEFAULT_THRESHOLD_CHARS = 20000;
 const DEFAULT_TARGET_CHARS = 15000;
 const DEFAULT_MODEL = "auto";
 const TARGET_BYTE_TOLERANCE = 0.02;
+const DREAM_COMPACTION_MAX_PASSES = 3;
+const DREAM_MODEL_MAX_ATTEMPTS = 3;
+const DREAM_MODEL_RETRY_DELAY_MS = 2000;
+// Manifest returns HTTP 200 with a "[🦚 Manifest <code>] ..." banner as the
+// assistant message when the router cannot fulfil a request (model unavailable,
+// overloaded, throttled). It is not JSON, so treat it as a retryable failure.
+const MANIFEST_BANNER_PATTERN = /^\s*\[\s*🦚/u;
+// Manifest surfaces upstream throttling and provider outages as HTTP errors.
+// A 429 is returned both for exhausted subscription quota and for a route that
+// is briefly cooling down, so retry these rather than failing the whole run.
+const RETRYABLE_HTTP_STATUSES = new Set([408, 429]);
 
 function numberFromEnv(name, fallback) {
   const value = process.env[name];
@@ -345,6 +356,37 @@ function createPrompt({ report, targetChars, threadEvidence }) {
   ];
 }
 
+function createCompactionRetryPrompt({ output, targetChars }) {
+  return [
+    {
+      role: "system",
+      content: [
+        "You are Hippocamp Dream, an offline memory curator.",
+        "The previous compaction pass was still too large.",
+        "Rewrite the complete snapshot again as a shorter, coherent snapshot.",
+        "Preserve the goal, important instructions, technical decisions, active state, and open work.",
+        "Remove repetition and historical detail before shortening individual facts.",
+        "Do not crop text, leave partial bullets, or invent facts.",
+        "Return strict JSON only, with keys current_state_md and open_threads_md.",
+        "current_state_md must start with '# Current State'.",
+        "open_threads_md must start with '# Open Threads'.",
+      ].join(" "),
+    },
+    {
+      role: "user",
+      content: [
+        `Target combined size for current_state.md and open_threads.md: ${targetChars} chars`,
+        "",
+        "## current_state.md",
+        output.currentState,
+        "",
+        "## open_threads.md",
+        output.openThreads,
+      ].join("\n"),
+    },
+  ];
+}
+
 function normalizeBaseUrl(value) {
   const trimmed = String(value || "").trim().replace(/\/+$/, "");
 
@@ -398,6 +440,18 @@ function getResponseText(payload) {
   return "";
 }
 
+function isManifestBanner(content) {
+  return MANIFEST_BANNER_PATTERN.test(content);
+}
+
+function isRetryableStatus(status) {
+  return RETRYABLE_HTTP_STATUSES.has(status) || status >= 500;
+}
+
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 async function callDreamModel({ apiKey, baseUrl, messages, model }) {
   if (!apiKey) {
     throw new Error("Missing API key. Set MANIFEST_API_KEY or pass --api-key.");
@@ -407,41 +461,90 @@ async function callDreamModel({ apiKey, baseUrl, messages, model }) {
     throw new Error("This command requires Node.js fetch support.");
   }
 
-  const response = await fetch(`${normalizeBaseUrl(baseUrl)}/responses`, {
-    method: "POST",
-    headers: {
-      Authorization: `Bearer ${apiKey}`,
-      "Content-Type": "application/json",
-    },
-    body: JSON.stringify({
-      input: createResponsesInput(messages),
-      model,
-      store: false,
-      text: createDreamTextFormat(),
-    }),
-  });
+  let lastBanner = null;
+  const endpoint = `${normalizeBaseUrl(baseUrl)}/responses`;
 
-  const text = await response.text();
+  for (let attempt = 1; attempt <= DREAM_MODEL_MAX_ATTEMPTS; attempt += 1) {
+    let response;
+    let text;
 
-  if (!response.ok) {
-    throw new Error(`Dream model request failed (${response.status}): ${text}`);
-  }
+    try {
+      response = await fetch(endpoint, {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${apiKey}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          input: createResponsesInput(messages),
+          model,
+          store: false,
+          text: createDreamTextFormat(),
+        }),
+      });
+      text = await response.text();
+    } catch (error) {
+      if (attempt < DREAM_MODEL_MAX_ATTEMPTS) {
+        await delay(DREAM_MODEL_RETRY_DELAY_MS * attempt);
+        continue;
+      }
 
-  const payload = JSON.parse(text);
-  const content = getResponseText(payload);
+      throw new Error(`Dream model request failed after ${DREAM_MODEL_MAX_ATTEMPTS} attempts: ${error.message}`);
+    }
+
+    if (!response.ok) {
+      if (isRetryableStatus(response.status) && attempt < DREAM_MODEL_MAX_ATTEMPTS) {
+        await delay(DREAM_MODEL_RETRY_DELAY_MS * attempt);
+        continue;
+      }
+
+      throw new Error(`Dream model request failed (${response.status}): ${text}`);
+    }
+
+    const payload = JSON.parse(text);
+    const content = getResponseText(payload);
+
+    if (typeof content !== "string" || !content.trim()) {
+      throw new Error("Dream model response did not include output text.");
+    }
+
+    if (isManifestBanner(content)) {
+      lastBanner = content.trim();
 
-  if (typeof content !== "string" || !content.trim()) {
-    throw new Error("Dream model response did not include output text.");
+      if (attempt < DREAM_MODEL_MAX_ATTEMPTS) {
+        await delay(DREAM_MODEL_RETRY_DELAY_MS * attempt);
+        continue;
+      }
+
+      throw new Error(
+        `Dream model returned a Manifest router banner instead of JSON after ${DREAM_MODEL_MAX_ATTEMPTS} attempts: ${lastBanner}`,
+      );
+    }
+
+    return content;
   }
 
-  return content;
+  // Unreachable: the loop either returns content or throws on the final attempt.
+  throw new Error(`Dream model returned a Manifest router banner instead of JSON: ${lastBanner}`);
 }
 
 function parseDreamJson(content) {
   const trimmed = content.trim();
   const match = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
   const jsonText = match ? match[1].trim() : trimmed;
-  const parsed = JSON.parse(jsonText);
+  let parsed;
+
+  try {
+    parsed = JSON.parse(jsonText);
+  } catch (error) {
+    const object = jsonText.match(/\{[\s\S]*\}/)?.[0];
+
+    if (!object) {
+      throw error;
+    }
+
+    parsed = JSON.parse(object);
+  }
 
   if (typeof parsed.current_state_md !== "string" || typeof parsed.open_threads_md !== "string") {
     throw new Error("Dream response must include current_state_md and open_threads_md strings.");
@@ -461,7 +564,7 @@ function getMaxOutputBytes(targetChars) {
   return Math.ceil(targetChars * (1 + TARGET_BYTE_TOLERANCE));
 }
 
-function validateDreamOutput(output, beforeBytes, targetChars) {
+function validateDreamStructure(output) {
   if (!output.currentState.startsWith("# Current State")) {
     throw new Error("Dream current_state_md must start with '# Current State'.");
   }
@@ -469,20 +572,44 @@ function validateDreamOutput(output, beforeBytes, targetChars) {
   if (!output.openThreads.startsWith("# Open Threads")) {
     throw new Error("Dream open_threads_md must start with '# Open Threads'.");
   }
+}
 
-  const afterBytes = getDreamOutputBytes(output);
-
-  if (afterBytes >= beforeBytes) {
-    throw new Error(`Dream output is not smaller than the input (${afterBytes} >= ${beforeBytes} bytes).`);
-  }
-
+async function compactDreamOutput({ apiKey, baseUrl, beforeBytes, model, report, targetChars, threadEvidence }) {
   const maxBytes = getMaxOutputBytes(targetChars);
+  let previousBytes = beforeBytes;
+  let previousOutput = null;
+
+  for (let pass = 1; pass <= DREAM_COMPACTION_MAX_PASSES; pass += 1) {
+    const messages = previousOutput
+      ? createCompactionRetryPrompt({ output: previousOutput, targetChars })
+      : createPrompt({ report, targetChars, threadEvidence });
+    const content = await callDreamModel({ apiKey, baseUrl, messages, model });
+    const output = parseDreamJson(content);
+    validateDreamStructure(output);
+
+    const afterBytes = getDreamOutputBytes(output);
+
+    if (afterBytes >= previousBytes) {
+      throw new Error(
+        `Dream compaction pass ${pass} did not reduce the snapshot (${afterBytes} >= ${previousBytes} bytes).`,
+      );
+    }
 
-  if (afterBytes > maxBytes) {
-    throw new Error(`Dream output exceeds --target-chars (${afterBytes} > ${maxBytes} bytes).`);
+    if (afterBytes <= maxBytes) {
+      return {
+        afterBytes,
+        output,
+        passes: pass,
+      };
+    }
+
+    previousBytes = afterBytes;
+    previousOutput = output;
   }
 
-  return afterBytes;
+  throw new Error(
+    `Dream output exceeds --target-chars after ${DREAM_COMPACTION_MAX_PASSES} coherent compaction passes (${previousBytes} > ${maxBytes} bytes).`,
+  );
 }
 
 async function writeDreamOutput(report, output) {
@@ -496,6 +623,7 @@ function toPublicReport(result) {
     afterWakeUpChars: result.afterWakeUpChars,
     beforeBytes: result.beforeBytes,
     changed: result.changed,
+    compactionPasses: result.compactionPasses,
     context: {
       evidenceBudgetChars: result.threadEvidence.evidenceBudgetChars,
       evidenceChars: result.threadEvidence.evidenceChars,
@@ -545,22 +673,24 @@ async function dreamProject(slug, options) {
     };
   }
 
-  const content = await callDreamModel({
+  const compacted = await compactDreamOutput({
     apiKey: options.apiKey,
     baseUrl: options.baseUrl,
-    messages: createPrompt({ report, targetChars: options.targetChars, threadEvidence }),
+    beforeBytes,
     model: options.model,
+    report,
+    targetChars: options.targetChars,
+    threadEvidence,
   });
-  const output = parseDreamJson(content);
-  const afterBytes = validateDreamOutput(output, beforeBytes, options.targetChars);
 
-  await writeDreamOutput(report, output);
+  await writeDreamOutput(report, compacted.output);
 
   return {
-    afterBytes,
+    afterBytes: compacted.afterBytes,
     afterWakeUpChars: (await memory.wakeUp({ projectRoot: getSyntheticProjectRoot(report.slug) })).text.length,
     beforeBytes,
     changed: true,
+    compactionPasses: compacted.passes,
     report,
     skipped: null,
     threadEvidence,
@@ -580,6 +710,7 @@ function printText(results) {
         `threads=${result.context.openThreads}`,
         `matchedThreads=${result.context.threadsWithEvidence}`,
         `evidence=${result.context.evidenceItems}`,
+        result.compactionPasses ? `passes=${result.compactionPasses}` : null,
         result.afterBytes ? `after=${result.afterBytes}` : null,
         result.afterWakeUpChars ? `afterWake=${result.afterWakeUpChars}` : null,
       ]
@@ -620,7 +751,18 @@ async function main() {
   printText(results);
 }
 
-main().catch((error) => {
-  console.error("Hippocamp Dream failed:", error.message);
-  process.exit(1);
-});
+if (require.main === module) {
+  main().catch((error) => {
+    console.error("Hippocamp Dream failed:", error.message);
+    process.exit(1);
+  });
+}
+
+module.exports = {
+  callDreamModel,
+  compactDreamOutput,
+  getDreamOutputBytes,
+  isManifestBanner,
+  isRetryableStatus,
+  parseDreamJson,
+};