nlm-memory 0.5.0 → 0.5.2

package/scripts/longmemeval/embedding-cache.ts

@@ -1,77 +0,0 @@
-/**
- * SHA256-keyed on-disk embedding cache. The LongMemEval-S haystack has
- * ~24K session bodies (~19K unique); embedding them via local Ollama takes
- * ~30 min the first time. Reruns must be instant — calibrating retrieval
- * parameters means dozens of re-evaluations, and re-embedding each time
- * would burn hours of wall clock for no signal.
- *
- * Backed by a small SQLite at $LONGMEMEVAL_CACHE_DIR/embeddings.sqlite.
- * Key = sha256(kind + ":" + text); value = Float32Array as BLOB.
- */
-
-import Database from "better-sqlite3";
-import type { Database as DB } from "better-sqlite3";
-import { createHash } from "node:crypto";
-import { mkdirSync } from "node:fs";
-import { dirname } from "node:path";
-import type { EmbeddingKind, LLMClient } from "../../src/ports/llm-client.js";
-
-const CREATE_SQL =
-  "CREATE TABLE IF NOT EXISTS embeddings (key TEXT PRIMARY KEY, vector BLOB NOT NULL)";
-
-export interface EmbeddingCacheOptions {
-  readonly dbPath: string;
-  readonly llm: LLMClient;
-}
-
-export class EmbeddingCache {
-  private readonly db: DB;
-  private readonly llm: LLMClient;
-  private readonly getStmt: ReturnType<DB["prepare"]>;
-  private readonly putStmt: ReturnType<DB["prepare"]>;
-
-  constructor(opts: EmbeddingCacheOptions) {
-    mkdirSync(dirname(opts.dbPath), { recursive: true });
-    this.db = new Database(opts.dbPath);
-    this.db.pragma("journal_mode = WAL");
-    this.db.prepare(CREATE_SQL).run();
-    this.getStmt = this.db.prepare(
-      "SELECT vector FROM embeddings WHERE key = @key",
-    );
-    this.putStmt = this.db.prepare(
-      "INSERT OR REPLACE INTO embeddings (key, vector) VALUES (@key, @vector)",
-    );
-    this.llm = opts.llm;
-  }
-
-  async embed(text: string, kind: EmbeddingKind): Promise<Float32Array> {
-    const key = createHash("sha256").update(`${kind}:${text}`).digest("hex");
-    const row = this.getStmt.get({ key }) as { vector: Buffer } | undefined;
-    if (row) {
-      return new Float32Array(
-        row.vector.buffer,
-        row.vector.byteOffset,
-        row.vector.byteLength / Float32Array.BYTES_PER_ELEMENT,
-      );
-    }
-    const result = await this.llm.embed(text, kind);
-    const blob = Buffer.from(
-      result.vector.buffer,
-      result.vector.byteOffset,
-      result.vector.byteLength,
-    );
-    this.putStmt.run({ key, vector: blob });
-    return result.vector;
-  }
-
-  size(): number {
-    const row = this.db.prepare("SELECT COUNT(*) AS n FROM embeddings").get() as {
-      n: number;
-    };
-    return row.n;
-  }
-
-  close(): void {
-    this.db.close();
-  }
-}

package/scripts/longmemeval/fetch-dataset.sh

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Fetch the LongMemEval-S (small) dataset from HuggingFace into a local cache.
-# Idempotent — skips download if the target file already exists with non-zero size.
-#
-# Source: https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned
-# Variant: longmemeval_s_cleaned.json — 500 questions × ~40 haystack sessions.
-
-set -euo pipefail
-
-CACHE_DIR="${LONGMEMEVAL_CACHE_DIR:-$HOME/.cache/longmemeval}"
-VARIANT="${LONGMEMEVAL_VARIANT:-longmemeval_s_cleaned.json}"
-URL="https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/${VARIANT}"
-TARGET="${CACHE_DIR}/${VARIANT}"
-
-mkdir -p "$CACHE_DIR"
-
-if [[ -s "$TARGET" ]]; then
-  echo "longmemeval-fetch: ${TARGET} already present ($(wc -c <"$TARGET") bytes) — skipping."
-  exit 0
-fi
-
-echo "longmemeval-fetch: downloading ${URL}"
-curl -fsSL --retry 3 -o "${TARGET}.tmp" "$URL"
-mv "${TARGET}.tmp" "$TARGET"
-echo "longmemeval-fetch: wrote ${TARGET} ($(wc -c <"$TARGET") bytes)"

package/scripts/longmemeval/run-harness.ts

@@ -1,315 +0,0 @@
-/**
- * LongMemEval-S baseline harness for NLM.
- *
- * Body-only ingest (skip classifier) + local Ollama nomic-embed-text. For
- * each evaluation instance: spin up an in-memory NLM corpus loaded with
- * the haystack sessions, query in each retrieval mode (keyword / semantic
- * / hybrid+RRF), score R@5 plus the session-body companion metric.
- *
- * Pure body-only retrieval — this measures the retrieval *algorithm*, not
- * the full classifier-in-loop NLM pipeline. The number is comparable to
- * agentmemory's published R@5 because both bench bodies-only.
- *
- * Usage:
- *   node dist/scripts/longmemeval/run-harness.js \
- *     --variant longmemeval_s_cleaned.json \
- *     --modes keyword,semantic,hybrid \
- *     --limit 500 \
- *     --report-dir reports/longmemeval
- *
- * Re-runs are fast: embeddings cache in ~/.cache/longmemeval/embeddings.sqlite
- * keyed by sha256(kind + text). First run = ~30 min embedding; subsequent = seconds.
- */
-
-import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
-import { homedir, tmpdir } from "node:os";
-import { dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-import { RecallService } from "../../src/core/recall/recall-service.js";
-import { SqliteSessionStore } from "../../src/core/storage/sqlite-session-store.js";
-import { OllamaClient } from "../../src/llm/ollama-client.js";
-import type {
-  EmbedResult,
-  EmbeddingKind,
-  LLMClient,
-} from "../../src/ports/llm-client.js";
-import type { RecallMode } from "../../src/shared/types.js";
-import { EmbeddingCache } from "./embedding-cache.js";
-import { scoreOne, aggregate, type SingleScore } from "./scorer.js";
-import { turnsToBody, type LongMemEvalInstance } from "./types.js";
-import { chunkSessionText } from "../../src/core/embedding/chunk-body.js";
-
-interface Args {
-  readonly datasetPath: string;
-  readonly modes: ReadonlyArray<RecallMode>;
-  readonly limit: number;
-  readonly k: number;
-  readonly reportDir: string;
-  readonly cacheDir: string;
-  readonly migrationsDir: string;
-}
-
-function parseArgs(argv: ReadonlyArray<string>): Args {
-  const get = (flag: string, fallback?: string): string => {
-    const i = argv.indexOf(flag);
-    if (i < 0) {
-      if (fallback === undefined) throw new Error(`missing required flag: ${flag}`);
-      return fallback;
-    }
-    return argv[i + 1] ?? "";
-  };
-  const cacheDir =
-    process.env["LONGMEMEVAL_CACHE_DIR"] ?? join(homedir(), ".cache", "longmemeval");
-  const variant = get("--variant", "longmemeval_s_cleaned.json");
-  const datasetPath = join(cacheDir, variant);
-  const modes = get("--modes", "keyword,semantic,hybrid")
-    .split(",")
-    .map((m) => m.trim()) as RecallMode[];
-  const limit = Number.parseInt(get("--limit", "500"), 10);
-  const k = Number.parseInt(get("--k", "5"), 10);
-  const reportDir = get("--report-dir", resolve("reports/longmemeval"));
-  const migrationsDir = resolve(__dirname, "../../migrations");
-  return { datasetPath, modes, limit, k, reportDir, cacheDir, migrationsDir };
-}
-
-/** LLMClient wrapper: routes embed() through the on-disk cache. */
-class CachingEmbedder implements LLMClient {
-  constructor(private readonly cache: EmbeddingCache) {}
-  async embed(text: string, kind: EmbeddingKind): Promise<EmbedResult> {
-    const vector = await this.cache.embed(text, kind);
-    return { vector, model: "nomic-embed-text@cached" };
-  }
-  async classify(): Promise<never> {
-    throw new Error("classify not used in LongMemEval body-only harness");
-  }
-}
-
-interface InstanceResult {
-  readonly question_id: string;
-  readonly question_type: string;
-  readonly by_mode: Record<string, SingleScore & { returnedIds: string[] }>;
-  readonly embed_failures: number;
-}
-
-async function runInstance(
-  instance: LongMemEvalInstance,
-  args: Args,
-  cache: EmbeddingCache,
-  embedder: LLMClient,
-): Promise<InstanceResult> {
-  const needsEmbeddings = args.modes.some(
-    (m) => m === "semantic" || m === "hybrid",
-  );
-  const tmpDir = mkdtempSync(join(tmpdir(), "nlm-lmeval-"));
-  const store = new SqliteSessionStore({
-    dbPath: join(tmpDir, "canonical.sqlite"),
-    migrationsDir: args.migrationsDir,
-  });
-  const bodyById = new Map<string, string>();
-  let embedFailures = 0;
-  const seen = new Set<string>();
-  try {
-    for (let i = 0; i < instance.haystack_sessions.length; i++) {
-      const id = instance.haystack_session_ids[i];
-      const date = instance.haystack_dates[i];
-      const turns = instance.haystack_sessions[i];
-      if (!id || !date || !turns) continue;
-      if (seen.has(id)) continue;
-      seen.add(id);
-      const body = turnsToBody(turns);
-      bodyById.set(id, body);
-      store.insertSessionForTest({
-        id,
-        runtime: "longmemeval",
-        runtimeSessionId: id,
-        startedAt: date,
-        endedAt: date,
-        durationMin: 0,
-        label: "",
-        summary: "",
-        body,
-        status: "closed",
-        transcriptKind: "longmemeval-jsonl",
-        transcriptPath: null,
-        entities: [],
-        decisions: [],
-        open: [],
-      });
-      if (needsEmbeddings) {
-        const chunks = chunkSessionText({ body });
-        for (let c = 0; c < chunks.length; c++) {
-          try {
-            const vector = await cache.embed(chunks[c]!, "document");
-            store.insertChunkEmbeddingForTest(id, c, vector);
-          } catch {
-            // Per-chunk embed failure is non-fatal — successfully embedded
-            // chunks still contribute via max-pool. Counter tracks attempts.
-            embedFailures++;
-          }
-        }
-      }
-    }
-
-    const recall = new RecallService({ store, llm: embedder });
-    const byMode: InstanceResult["by_mode"] = {};
-    for (const mode of args.modes) {
-      const result = await recall.search({
-        query: instance.question,
-        mode,
-        limit: args.k,
-      });
-      const returnedIds = result.results.map((r) => r.id);
-      const returnedBodies = returnedIds.map((id) => bodyById.get(id) ?? "");
-      const score = scoreOne({
-        returnedIds,
-        goldIds: instance.answer_session_ids,
-        returnedBodies,
-        answer: instance.answer,
-        k: args.k,
-      });
-      byMode[mode] = { ...score, returnedIds };
-    }
-    return {
-      question_id: instance.question_id,
-      question_type: instance.question_type,
-      by_mode: byMode,
-      embed_failures: embedFailures,
-    };
-  } finally {
-    store.close();
-    rmSync(tmpDir, { recursive: true, force: true });
-  }
-}
-
-async function main(): Promise<void> {
-  const args = parseArgs(process.argv.slice(2));
-  console.log(`longmemeval-harness: loading ${args.datasetPath}`);
-  const raw = readFileSync(args.datasetPath, "utf8");
-  const dataset = JSON.parse(raw) as LongMemEvalInstance[];
-  const slice = dataset.slice(0, args.limit);
-  console.log(
-    `longmemeval-harness: ${slice.length}/${dataset.length} instances, modes=${args.modes.join(",")}, k=${args.k}`,
-  );
-
-  // Warm the embedding cache and the LLM transport.
-  const llm = new OllamaClient({ embedModel: "nomic-embed-text" });
-  const cache = new EmbeddingCache({
-    dbPath: join(args.cacheDir, "embeddings.sqlite"),
-    llm,
-  });
-  const embedder = new CachingEmbedder(cache);
-
-  console.log(`longmemeval-harness: cache contains ${cache.size()} embeddings on entry`);
-
-  const results: InstanceResult[] = [];
-  const t0 = Date.now();
-  for (let i = 0; i < slice.length; i++) {
-    const inst = slice[i];
-    if (!inst) continue;
-    const result = await runInstance(inst, args, cache, embedder);
-    results.push(result);
-    if ((i + 1) % 10 === 0 || i === slice.length - 1) {
-      const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
-      const cached = cache.size();
-      console.log(
-        `  [${i + 1}/${slice.length}] ${elapsed}s elapsed, cache=${cached}`,
-      );
-    }
-  }
-
-  // Aggregate.
-  const aggregated: Record<string, ReturnType<typeof aggregate>> = {};
-  for (const mode of args.modes) {
-    aggregated[mode] = aggregate(
-      results.map((r) => r.by_mode[mode] as SingleScore).filter(Boolean),
-    );
-  }
-
-  // Per-question-type breakdown.
-  const byType: Record<string, Record<string, ReturnType<typeof aggregate>>> = {};
-  const types = new Set(results.map((r) => r.question_type));
-  for (const t of types) {
-    byType[t] = {};
-    for (const mode of args.modes) {
-      const subset = results
-        .filter((r) => r.question_type === t)
-        .map((r) => r.by_mode[mode] as SingleScore)
-        .filter(Boolean);
-      byType[t]![mode] = aggregate(subset);
-    }
-  }
-
-  const stamp = new Date().toISOString().replace(/[:T]/g, "-").slice(0, 19);
-  const outDir = join(args.reportDir, stamp);
-  mkdirSync(outDir, { recursive: true });
-  const json = {
-    dataset: args.datasetPath,
-    n: results.length,
-    k: args.k,
-    modes: args.modes,
-    aggregate: aggregated,
-    by_question_type: byType,
-    results,
-    elapsed_seconds: (Date.now() - t0) / 1000,
-  };
-  writeFileSync(join(outDir, "results.json"), JSON.stringify(json, null, 2));
-  writeFileSync(join(outDir, "summary.md"), renderSummary(json));
-  console.log(`longmemeval-harness: wrote ${outDir}/`);
-  console.log(renderSummary(json));
-
-  cache.close();
-}
-
-function renderSummary(json: {
-  dataset: string;
-  n: number;
-  k: number;
-  modes: ReadonlyArray<RecallMode>;
-  aggregate: Record<string, ReturnType<typeof aggregate>>;
-  by_question_type: Record<string, Record<string, ReturnType<typeof aggregate>>>;
-  elapsed_seconds: number;
-}): string {
-  const lines: string[] = [];
-  lines.push(`# LongMemEval-S — NLM baseline (body-only, n=${json.n}, k=${json.k})`);
-  lines.push("");
-  lines.push(`Dataset: \`${json.dataset}\``);
-  lines.push(`Elapsed: ${json.elapsed_seconds.toFixed(1)}s`);
-  lines.push("");
-  lines.push("## Aggregate");
-  lines.push("");
-  lines.push(`| Mode | R@${json.k} | Session-body hit |`);
-  lines.push(`| --- | --- | --- |`);
-  for (const mode of json.modes) {
-    const a = json.aggregate[mode];
-    if (!a) continue;
-    lines.push(
-      `| ${mode} | ${(a.recallAtK * 100).toFixed(1)}% | ${(a.sessionBodyHitRate * 100).toFixed(1)}% |`,
-    );
-  }
-  lines.push("");
-  lines.push("## By question type");
-  lines.push("");
-  const types = Object.keys(json.by_question_type).sort();
-  lines.push(
-    `| Question type | ${json.modes.map((m) => `${m} R@${json.k}`).join(" | ")} |`,
-  );
-  lines.push(`| --- | ${json.modes.map(() => "---").join(" | ")} |`);
-  for (const t of types) {
-    const row = json.by_question_type[t]!;
-    const cells = json.modes.map((m) => {
-      const a = row[m];
-      return a ? `${(a.recallAtK * 100).toFixed(1)}% (n=${a.n})` : "—";
-    });
-    lines.push(`| ${t} | ${cells.join(" | ")} |`);
-  }
-  lines.push("");
-  return lines.join("\n");
-}
-
-void main().catch((err) => {
-  console.error("longmemeval-harness: fatal", err);
-  process.exit(1);
-});

package/scripts/longmemeval/scorer.ts

@@ -1,99 +0,0 @@
-/**
- * Pure scoring functions for the LongMemEval harness. Two metrics:
- *
- *  - R@k (recall at k): did the retriever return any gold session ID in
- *    its top-k results? Standard benchmark metric.
- *  - Session-body hit: did the gold answer text appear anywhere in the
- *    bodies of the top-k returned sessions? NLM-specific companion that
- *    captures session-as-primary-unit value the strict-ID R@k can miss
- *    (e.g. a session that supersedes the gold session and quotes its
- *    decision).
- *
- * Both functions are deterministic and dependency-free so the harness can
- * test them with synthetic inputs.
- */
-
-export interface ScoreInputs {
-  readonly returnedIds: ReadonlyArray<string>;
-  readonly goldIds: ReadonlyArray<string>;
-  /** Map id → body for the bodies of the top-k returned sessions. */
-  readonly returnedBodies: ReadonlyArray<string>;
-  /** Some LongMemEval answers are ints (counting questions); coerced to string. */
-  readonly answer: string | number | boolean;
-  readonly k: number;
-}
-
-export interface SingleScore {
-  readonly recallAtK: 0 | 1;
-  readonly sessionBodyHit: 0 | 1;
-}
-
-/** Score a single question. Returns 0/1 indicators that aggregate via mean. */
-export function scoreOne(input: ScoreInputs): SingleScore {
-  const topK = input.returnedIds.slice(0, input.k);
-  const goldSet = new Set(input.goldIds);
-  const recallAtK = topK.some((id) => goldSet.has(id)) ? 1 : 0;
-
-  // Session-body hit: substring match for multi-word answers; word-boundary
-  // match for short answers (single token <4 chars: "3", "yes", numeric
-  // counts). Without the boundary, a numeric answer "3" hits every body
-  // containing "3 days", "$3", etc., inflating the metric to noise.
-  const ans = normalize(String(input.answer));
-  let sessionBodyHit: 0 | 1 = 0;
-  if (ans.length > 0) {
-    const isShortToken = !ans.includes(" ") && ans.length < 4;
-    const test = isShortToken
-      ? (body: string): boolean =>
-          new RegExp(`\\b${escapeRegExp(ans)}\\b`).test(normalize(body))
-      : (body: string): boolean => normalize(body).includes(ans);
-    const bodies = input.returnedBodies.slice(0, input.k);
-    for (const body of bodies) {
-      if (test(body)) {
-        sessionBodyHit = 1;
-        break;
-      }
-    }
-  }
-  return { recallAtK, sessionBodyHit };
-}
-
-export interface Aggregate {
-  readonly n: number;
-  readonly recallAtK: number;
-  readonly sessionBodyHitRate: number;
-}
-
-/** Aggregate per-question scores into mean rates. */
-export function aggregate(scores: ReadonlyArray<SingleScore>): Aggregate {
-  const n = scores.length;
-  if (n === 0) {
-    return { n: 0, recallAtK: 0, sessionBodyHitRate: 0 };
-  }
-  let r = 0;
-  let s = 0;
-  for (const x of scores) {
-    r += x.recallAtK;
-    s += x.sessionBodyHit;
-  }
-  return {
-    n,
-    recallAtK: round3(r / n),
-    sessionBodyHitRate: round3(s / n),
-  };
-}
-
-function normalize(s: string): string {
-  return s
-    .toLowerCase()
-    .replace(/[^\p{L}\p{N}\s]/gu, " ")
-    .replace(/\s+/g, " ")
-    .trim();
-}
-
-function round3(x: number): number {
-  return Math.round(x * 1000) / 1000;
-}
-
-function escapeRegExp(s: string): string {
-  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}

package/scripts/longmemeval/tsconfig.json

@@ -1,9 +0,0 @@
-{
-  "extends": "../../tsconfig.json",
-  "compilerOptions": {
-    "rootDir": "../..",
-    "outDir": "../../dist-scripts",
-    "noEmit": true
-  },
-  "include": ["./*.ts", "../../src/**/*.ts"]
-}

package/scripts/longmemeval/types.ts

@@ -1,35 +0,0 @@
-/**
- * LongMemEval dataset schema. Mirrors the published JSON shape from
- * huggingface.co/datasets/xiaowu0162/longmemeval-cleaned.
- *
- * Each instance: a question against a haystack of past chat sessions. The
- * gold session IDs are in `answer_session_ids` — that's what the retrieval
- * step is scored against (R@k: was any gold ID returned in the top k).
- */
-
-export interface LongMemEvalTurn {
-  readonly role: "user" | "assistant";
-  readonly content: string;
-  readonly has_answer?: boolean;
-}
-
-export interface LongMemEvalInstance {
-  readonly question_id: string;
-  readonly question_type: string;
-  readonly question: string;
-  // LongMemEval answers are sometimes ints/booleans for counting and
-  // temporal-reasoning questions — coerce at the call site.
-  readonly answer: string | number | boolean;
-  readonly question_date: string;
-  readonly haystack_session_ids: ReadonlyArray<string>;
-  readonly haystack_dates: ReadonlyArray<string>;
-  readonly haystack_sessions: ReadonlyArray<ReadonlyArray<LongMemEvalTurn>>;
-  readonly answer_session_ids: ReadonlyArray<string>;
-}
-
-/** Serialize a session's turn list to a single body string for NLM ingest. */
-export function turnsToBody(turns: ReadonlyArray<LongMemEvalTurn>): string {
-  return turns
-    .map((t) => `${t.role === "user" ? "User" : "Assistant"}: ${t.content}`)
-    .join("\n\n");
-}