nlm-memory 0.4.2 → 0.5.1

package/src/core/embedding/chunk-body.ts

@@ -1,76 +0,0 @@
-/**
- * chunk-body — split a session body into ≤MAX_CHUNK_CHARS-char windows
- * for the chunk + max-pool semantic index. Header (label + summary) is
- * prepended to the first chunk so it's always part of the index without
- * inflating later chunk sizes.
- *
- * MAX_CHUNK_CHARS sized for nomic-embed-text's 2048-token context. Char
- * density varies by content: prose ~4 chars/token, code/tool-output ~3
- * chars/token. The 2026-05-26 backfill bisect found the cliff at ~6,388
- * chars for token-dense Claude Code session bodies — 5,500 holds a safe
- * margin and eliminates the "input exceeds context length" 500s that
- * drove ~76% per-chunk rejection at 7,500. See 2026-05-26 CHANGELOG.
- *
- * OVERLAP_CHARS preserves context across boundaries so a phrase split
- * mid-chunk still appears intact in one neighboring chunk.
- *
- * Pure function. No I/O, no allocations beyond the returned array.
- */
-
-export const MAX_CHUNK_CHARS = 5_500;
-export const OVERLAP_CHARS = 500;
-
-export interface ChunkInput {
-  readonly label?: string | null;
-  readonly summary?: string | null;
-  readonly body?: string | null;
-}
-
-export interface ChunkOptions {
-  readonly maxChars?: number;
-  readonly overlap?: number;
-}
-
-export function chunkSessionText(
-  input: ChunkInput,
-  opts: ChunkOptions = {},
-): string[] {
-  const maxChars = opts.maxChars ?? MAX_CHUNK_CHARS;
-  const overlap = opts.overlap ?? OVERLAP_CHARS;
-  if (maxChars <= 0) throw new Error("chunkSessionText: maxChars must be > 0");
-  if (overlap < 0 || overlap >= maxChars) {
-    throw new Error("chunkSessionText: overlap must satisfy 0 <= overlap < maxChars");
-  }
-
-  const header = [input.label ?? "", input.summary ?? ""]
-    .map((s) => s.trim())
-    .filter((s) => s.length > 0)
-    .join(" ");
-  const body = (input.body ?? "").trim();
-
-  if (!header && !body) return [];
-  if (!body) return [header];
-
-  // First chunk: header + as much body as fits.
-  const headerPrefix = header ? header + " " : "";
-  const firstBodyBudget = Math.max(1, maxChars - headerPrefix.length);
-
-  if (body.length <= firstBodyBudget) {
-    return [(headerPrefix + body).trim()];
-  }
-
-  const chunks: string[] = [];
-  chunks.push((headerPrefix + body.slice(0, firstBodyBudget)).trim());
-
-  // Subsequent chunks: body windows with overlap.
-  const step = maxChars - overlap;
-  let pos = Math.max(0, firstBodyBudget - overlap);
-  while (pos < body.length) {
-    const end = Math.min(pos + maxChars, body.length);
-    const slice = body.slice(pos, end).trim();
-    if (slice.length > 0) chunks.push(slice);
-    if (end >= body.length) break;
-    pos += step;
-  }
-  return chunks;
-}

package/src/core/embedding/embed-backfill.ts

@@ -1,210 +0,0 @@
-/**
- * embed-backfill — re-embed every session in canonical.sqlite into the
- * chunk + max-pool index (session_embedding_chunks). Replaces the prior
- * one-vector-per-session backfill that wrote to session_embeddings.
- *
- * For each session: chunk (label + summary + body) via chunkSessionText,
- * embed each chunk with kind="document", and write to the chunk table +
- * session_chunk_map via the same INSERT pair used by the live ingest path.
- *
- * Resumable via a JSON state file at $NLM_EMBED_STATE (default
- * ~/.nlm/embed_reembed.state). Interrupting + rerunning skips already-done
- * session ids. A session is considered "done" only when ALL its chunks
- * embed successfully — partial sessions are retried on the next run.
- *
- * Layering: depends on the LLMClient port. SQLite touched directly via
- * better-sqlite3 because this is a one-shot operational tool, not a hot
- * path. Lives under core/ but is invoked from the CLI composition root.
- */
-
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { homedir } from "node:os";
-import Database from "better-sqlite3";
-import * as sqliteVec from "sqlite-vec";
-import type { LLMClient } from "@ports/llm-client.js";
-import { LLMUnreachableError } from "@ports/llm-client.js";
-import { chunkSessionText } from "@core/embedding/chunk-body.js";
-
-const DEFAULT_STATE_PATH = join(homedir(), ".nlm", "embed_reembed.state");
-const SAVE_EVERY = 25;
-
-export interface BackfillOptions {
-  readonly dbPath: string;
-  readonly embedder: LLMClient;
-  readonly statePath?: string;
-  readonly limit?: number;
-  readonly onProgress?: (i: number, total: number, sid: string, status: string) => void;
-}
-
-export interface BackfillReport {
-  readonly total: number;
-  readonly processed: number;
-  readonly succeeded: number;
-  readonly failed: number;
-  readonly skippedAlreadyDone: number;
-  readonly dbMissing: boolean;
-}
-
-interface SessionRow {
-  id: string;
-  label: string | null;
-  summary: string | null;
-  body: string | null;
-}
-
-function loadState(path: string): Set<string> {
-  if (!existsSync(path)) return new Set();
-  try {
-    const data = JSON.parse(readFileSync(path, "utf8")) as { done?: string[] };
-    return new Set(data.done ?? []);
-  } catch {
-    return new Set();
-  }
-}
-
-function saveState(path: string, done: Set<string>): void {
-  mkdirSync(dirname(path), { recursive: true });
-  writeFileSync(path, JSON.stringify({ done: [...done].sort() }));
-}
-
-
-export async function reembedCorpus(opts: BackfillOptions): Promise<BackfillReport> {
-  const statePath = opts.statePath ?? DEFAULT_STATE_PATH;
-
-  if (!existsSync(opts.dbPath)) {
-    return { total: 0, processed: 0, succeeded: 0, failed: 0, skippedAlreadyDone: 0, dbMissing: true };
-  }
-
-  const db = new Database(opts.dbPath);
-  sqliteVec.load(db);
-
-  // Backfill every session with content; live ingest covers ongoing writes.
-  // The state file dedupes across runs so partial completion resumes cleanly.
-  const sql =
-    "SELECT s.id, s.label, s.summary, s.body FROM sessions s " +
-    "WHERE s.body IS NOT NULL OR s.summary IS NOT NULL OR s.label IS NOT NULL " +
-    "ORDER BY s.started_at" +
-    (opts.limit ? ` LIMIT ${Math.trunc(opts.limit)}` : "");
-  const rows = db.prepare<[], SessionRow>(sql).all();
-  const total = rows.length;
-
-  const done = loadState(statePath);
-
-  const selectChunks = db.prepare<[string], { chunk_id: number }>(
-    "SELECT chunk_id FROM session_chunk_map WHERE session_id = ?",
-  );
-  const delChunks = (sessionId: string): void => {
-    const existing = selectChunks.all(sessionId);
-    if (existing.length === 0) return;
-    const placeholders = existing.map(() => "?").join(",");
-    const ids = existing.map((r) => r.chunk_id);
-    db.prepare(
-      `DELETE FROM session_embedding_chunks WHERE chunk_id IN (${placeholders})`,
-    ).run(...ids);
-    db.prepare("DELETE FROM session_chunk_map WHERE session_id = ?").run(sessionId);
-  };
-  const insChunk = db.prepare(
-    "INSERT INTO session_embedding_chunks (embedding, session_id, chunk_idx) VALUES (?, ?, ?)",
-  );
-  const insMap = db.prepare(
-    "INSERT INTO session_chunk_map (chunk_id, session_id, chunk_idx) VALUES (?, ?, ?)",
-  );
-
-  let succeeded = 0;
-  let failed = 0;
-  let skipped = 0;
-
-  try {
-    for (let i = 0; i < rows.length; i++) {
-      const row = rows[i]!;
-      const idx = i + 1;
-      if (done.has(row.id)) {
-        skipped += 1;
-        continue;
-      }
-      const chunks = chunkSessionText({
-        label: row.label,
-        summary: row.summary,
-        body: row.body,
-      });
-      if (chunks.length === 0) {
-        failed += 1;
-        opts.onProgress?.(idx, total, row.id, "SKIP (no text)");
-        continue;
-      }
-
-      // Per-chunk failure tolerance matches live ingest: one chunk hitting
-      // the Ollama edge-cliff 500 must not zero out an entire session's
-      // coverage. Single retry on LLMUnreachableError catches transient
-      // failures; persistent ones are dropped. Session is "done" if any
-      // chunk landed — partial max-pool coverage beats none.
-      const vectors: { idx: number; vec: Float32Array }[] = [];
-      let chunkSkipped = 0;
-      for (let c = 0; c < chunks.length; c++) {
-        const chunk = chunks[c]!;
-        let lastErr: unknown;
-        for (let attempt = 0; attempt < 2; attempt++) {
-          try {
-            const out = await opts.embedder.embed(chunk, "document");
-            vectors.push({ idx: c, vec: out.vector });
-            lastErr = undefined;
-            break;
-          } catch (e) {
-            lastErr = e;
-            if (!(e instanceof LLMUnreachableError)) throw e;
-            if (attempt === 0) await new Promise((r) => setTimeout(r, 200));
-          }
-        }
-        if (lastErr !== undefined) chunkSkipped += 1;
-      }
-      if (vectors.length === 0) {
-        failed += 1;
-        opts.onProgress?.(idx, total, row.id, `FAIL (embedder, ${chunkSkipped}/${chunks.length} chunks)`);
-        continue;
-      }
-
-      try {
-        delChunks(row.id);
-        for (const { idx: cidx, vec } of vectors) {
-          const blob = Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
-          // BigInt cast so vec0's aux chunk_idx column receives an INTEGER.
-          const info = insChunk.run(blob, row.id, BigInt(cidx));
-          insMap.run(Number(info.lastInsertRowid), row.id, cidx);
-        }
-      } catch (e) {
-        failed += 1;
-        opts.onProgress?.(idx, total, row.id, `FAIL (db): ${(e as Error).message}`);
-        continue;
-      }
-
-      done.add(row.id);
-      succeeded += 1;
-      const status =
-        chunkSkipped === 0
-          ? `OK (${vectors.length} chunks)`
-          : `PARTIAL (${vectors.length}/${chunks.length} chunks, ${chunkSkipped} skipped)`;
-      opts.onProgress?.(idx, total, row.id, status);
-      if (succeeded % SAVE_EVERY === 0) saveState(statePath, done);
-    }
-    saveState(statePath, done);
-  } finally {
-    db.close();
-  }
-
-  return {
-    total,
-    processed: succeeded + failed + skipped,
-    succeeded,
-    failed,
-    skippedAlreadyDone: skipped,
-    dbMissing: false,
-  };
-}
-
-export function clearBackfillState(statePath: string = DEFAULT_STATE_PATH): void {
-  if (existsSync(statePath)) {
-    const { unlinkSync } = require("node:fs") as typeof import("node:fs");
-    unlinkSync(statePath);
-  }
-}

package/src/core/embedding/embed-normalize.ts

@@ -1,135 +0,0 @@
-/**
- * embed-normalize — one-shot migration: L2-normalize every row in
- * session_embeddings. Ports `embed_normalize.py`.
- *
- * vec0 with implicit L2 distance ranks correctly by cosine similarity
- * only when stored vectors are unit-length. New writes (post-this-fix)
- * are normalized at source by OllamaClient.embed; this module brings
- * existing rows to the same invariant.
- *
- * Idempotent: re-running on already-normalized vectors is a no-op
- * within float tolerance (EPS = 1e-3). Each row is rewritten in its
- * own transaction so interrupts are safe.
- */
-
-import { existsSync } from "node:fs";
-import Database from "better-sqlite3";
-import * as sqliteVec from "sqlite-vec";
-
-const EPS = 1e-3;
-const DEFAULT_DIM = 768;
-const DEFAULT_BATCH = 100;
-
-export interface NormalizeOptions {
-  readonly dbPath: string;
-  readonly dim?: number;
-  readonly batchSize?: number;
-  readonly dryRun?: boolean;
-}
-
-export interface NormalizeReport {
-  readonly total: number;
-  readonly alreadyNormalized: number;
-  readonly rewritten: number;
-  readonly zeroVector: number;
-  readonly dbMissing: boolean;
-  readonly dryRun: boolean;
-}
-
-interface EmbeddingRow {
-  session_id: string;
-  embedding: Buffer;
-}
-
-interface IdRow {
-  session_id: string;
-}
-
-function bytesToFloats(buf: Buffer, dim: number): Float32Array {
-  if (buf.byteLength !== dim * 4) {
-    throw new Error(`expected ${dim * 4} bytes, got ${buf.byteLength}`);
-  }
-  return new Float32Array(buf.buffer, buf.byteOffset, dim);
-}
-
-function floatsToBytes(vec: Float32Array): Buffer {
-  return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
-}
-
-export function normalizeEmbeddings(opts: NormalizeOptions): NormalizeReport {
-  const dim = opts.dim ?? DEFAULT_DIM;
-  const batchSize = opts.batchSize ?? DEFAULT_BATCH;
-  const dryRun = opts.dryRun ?? false;
-
-  if (!existsSync(opts.dbPath)) {
-    return { total: 0, alreadyNormalized: 0, rewritten: 0, zeroVector: 0, dbMissing: true, dryRun };
-  }
-
-  const db = new Database(opts.dbPath);
-  sqliteVec.load(db);
-
-  let total = 0;
-  let alreadyNormalized = 0;
-  let rewritten = 0;
-  let zeroVector = 0;
-
-  try {
-    const ids = db
-      .prepare<[], IdRow>("SELECT session_id FROM session_embeddings")
-      .all()
-      .map((r) => r.session_id);
-    total = ids.length;
-
-    const sel = db.prepare<[string], EmbeddingRow>(
-      "SELECT session_id, embedding FROM session_embeddings WHERE session_id = ?",
-    );
-    const del = db.prepare("DELETE FROM session_embeddings WHERE session_id = ?");
-    const ins = db.prepare("INSERT INTO session_embeddings (session_id, embedding) VALUES (?, ?)");
-
-    for (let start = 0; start < total; start += batchSize) {
-      const batch = ids.slice(start, start + batchSize);
-      for (const sid of batch) {
-        const row = sel.get(sid);
-        if (!row) continue;
-
-        const vec = bytesToFloats(row.embedding, dim);
-        let sumSq = 0;
-        for (let i = 0; i < dim; i++) {
-          const v = vec[i] ?? 0;
-          sumSq += v * v;
-        }
-        if (sumSq === 0) {
-          zeroVector += 1;
-          continue;
-        }
-        const norm = Math.sqrt(sumSq);
-        if (Math.abs(norm - 1) <= EPS) {
-          alreadyNormalized += 1;
-          continue;
-        }
-        if (dryRun) {
-          rewritten += 1;
-          continue;
-        }
-        const normalized = new Float32Array(dim);
-        for (let i = 0; i < dim; i++) {
-          normalized[i] = (vec[i] ?? 0) / norm;
-        }
-        del.run(sid);
-        ins.run(sid, floatsToBytes(normalized));
-        rewritten += 1;
-      }
-    }
-  } finally {
-    db.close();
-  }
-
-  return {
-    total,
-    alreadyNormalized,
-    rewritten,
-    zeroVector,
-    dbMissing: false,
-    dryRun,
-  };
-}

package/src/core/facts/backfill-facts.ts

@@ -1,254 +0,0 @@
-/**
- * backfill-facts — one-shot population of the FactStore from the existing
- * session corpus. Phase B.5, see docs/plans/factstore-design.md Section 7.
- *
- * For each session in `sessions` that has no facts yet (and was started
- * before the script's start timestamp, to avoid racing with live ingest),
- * runs the classifier over its body, extracts facts, and writes them via
- * SqliteSessionStore.insertFactsForSession.
- *
- * Resumable via a JSON state file (mirrors core/embedding/embed-backfill).
- * Interrupting and rerunning skips already-processed sessions. State path
- * defaults to ~/.nlm/backfill_facts.state.
- *
- * Layering: depends on the LLMClient + FactStore ports through the
- * SqliteSessionStore + SqliteFactStore composition. Lives under core/ but
- * is invoked from the CLI composition root, like embed-backfill.
- */
-
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { homedir } from "node:os";
-import { extractFacts } from "@core/facts/extract-facts.js";
-import type { SqliteFactStore } from "@core/storage/sqlite-fact-store.js";
-import type { SqliteSessionStore } from "@core/storage/sqlite-session-store.js";
-import type { LLMClient } from "@ports/llm-client.js";
-import { LLMUnreachableError } from "@ports/llm-client.js";
-
-const DEFAULT_STATE_PATH = join(homedir(), ".nlm", "backfill_facts.state");
-const SAVE_EVERY = 25;
-
-export interface BackfillFactsOptions {
-  readonly store: SqliteSessionStore;
-  readonly factStore: SqliteFactStore;
-  readonly classifier: LLMClient;
-  /** Optional embedder. When omitted, facts are written without semantic vectors. */
-  readonly embedder?: LLMClient | null;
-  readonly statePath?: string;
-  /** Cap on sessions processed this run. Default: all eligible. */
-  readonly limit?: number;
-  /**
-   * Resume from a specific session id. When set, sessions with id
-   * lexicographically <= this value are skipped on top of the state file's
-   * done set. Useful when the state file is lost but the operator
-   * remembers the last successful id.
-   */
-  readonly from?: string;
-  /** Don't write — just count what would happen. */
-  readonly dryRun?: boolean;
-  /**
-   * Re-process sessions that already have facts. Default: false (skip).
-   * Use when iterating the classifier prompt to refresh the corpus.
-   */
-  readonly reprocess?: boolean;
-  readonly onProgress?: (
-    i: number,
-    total: number,
-    sessionId: string,
-    status: BackfillStatus,
-    details?: string,
-  ) => void;
-}
-
-export type BackfillStatus =
-  | "ok"
-  | "skipped_done"
-  | "skipped_existing_facts"
-  | "skipped_no_body"
-  | "skipped_low_confidence"
-  | "classify_failed"
-  | "storage_failed";
-
-export interface BackfillFactsReport {
-  readonly total: number;
-  readonly processed: number;
-  readonly factsWritten: number;
-  readonly skippedAlreadyDone: number;
-  readonly skippedExistingFacts: number;
-  readonly skippedNoBody: number;
-  readonly skippedLowConfidence: number;
-  readonly classifyFailures: number;
-  readonly storageFailures: number;
-}
-
-interface CandidateRow {
-  id: string;
-  started_at: string;
-  body: string | null;
-}
-
-function loadState(path: string): Set<string> {
-  if (!existsSync(path)) return new Set();
-  try {
-    const data = JSON.parse(readFileSync(path, "utf8")) as { done?: string[] };
-    return new Set(data.done ?? []);
-  } catch {
-    return new Set();
-  }
-}
-
-function saveState(path: string, done: Set<string>): void {
-  const dir = dirname(path);
-  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
-  writeFileSync(path, JSON.stringify({ done: Array.from(done) }, null, 0));
-}
-
-export async function backfillFacts(
-  opts: BackfillFactsOptions,
-): Promise<BackfillFactsReport> {
-  const startedAtCutoff = new Date().toISOString();
-  const statePath = opts.statePath ?? DEFAULT_STATE_PATH;
-  const done = opts.dryRun ? new Set<string>() : loadState(statePath);
-
-  const db = opts.store.rawDb();
-
-  // Eligible sessions: started strictly before this run's cutoff (don't
-  // race with live ingest), with a non-empty body (the classifier needs
-  // transcript text). When reprocess=false, exclude sessions that already
-  // have facts attributed to them.
-  const sql = opts.reprocess
-    ? `
-      SELECT id, started_at, body
-      FROM sessions
-      WHERE started_at < ?
-        AND body IS NOT NULL AND length(body) > 0
-        ${opts.from ? "AND id > ?" : ""}
-      ORDER BY started_at ASC, id ASC
-    `
-    : `
-      SELECT s.id, s.started_at, s.body
-      FROM sessions s
-      WHERE s.started_at < ?
-        AND s.body IS NOT NULL AND length(s.body) > 0
-        AND NOT EXISTS (
-          SELECT 1 FROM facts f WHERE f.source_session_id = s.id
-        )
-        ${opts.from ? "AND s.id > ?" : ""}
-      ORDER BY s.started_at ASC, s.id ASC
-    `;
-  const rows: CandidateRow[] = opts.from
-    ? db.prepare<[string, string], CandidateRow>(sql).all(startedAtCutoff, opts.from)
-    : db.prepare<[string], CandidateRow>(sql).all(startedAtCutoff);
-
-  // Filter state-file-known done ids BEFORE applying limit. Without this,
-  // a dense cluster of previously-skipped (low-confidence) sessions would
-  // burn the batch's --limit on no-op skips. With it, --limit N means
-  // "N actually-processable sessions" — much more useful UX for repeated
-  // small batches that walk forward through the corpus. The pre-filter
-  // count gets reported as `skippedAlreadyDone` so the operator still sees
-  // how big the skip region was.
-  const skippedByStateFile = rows.filter((r) => done.has(r.id)).length;
-  const candidates = rows.filter((r) => !done.has(r.id));
-  const limit = opts.limit ?? candidates.length;
-  const work = candidates.slice(0, limit);
-  const total = work.length;
-
-  let processed = 0;
-  let factsWritten = 0;
-  let skippedAlreadyDone = skippedByStateFile;
-  let skippedExistingFacts = 0;
-  let skippedNoBody = 0;
-  let skippedLowConfidence = 0;
-  let classifyFailures = 0;
-  let storageFailures = 0;
-
-  for (let i = 0; i < work.length; i++) {
-    const row = work[i]!;
-    const sid = row.id;
-
-    // No per-iteration `done` check needed — `work` is already filtered
-    // against the state file above.
-
-    if (!row.body || row.body.length === 0) {
-      skippedNoBody += 1;
-      opts.onProgress?.(i + 1, total, sid, "skipped_no_body");
-      continue;
-    }
-
-    let classification;
-    try {
-      classification = await opts.classifier.classify(row.body);
-    } catch (err) {
-      classifyFailures += 1;
-      const detail =
-        err instanceof LLMUnreachableError
-          ? "ollama unreachable — stopping run"
-          : err instanceof Error
-            ? err.message
-            : String(err);
-      opts.onProgress?.(i + 1, total, sid, "classify_failed", detail);
-      // Ollama-down is fatal: every subsequent classify will fail. Stop
-      // here so the operator can fix and resume.
-      if (err instanceof LLMUnreachableError) break;
-      continue;
-    }
-
-    const facts = extractFacts(classification, sid, row.started_at);
-    if (facts.length === 0) {
-      skippedLowConfidence += 1;
-      opts.onProgress?.(
-        i + 1,
-        total,
-        sid,
-        "skipped_low_confidence",
-        `confidence=${classification.confidence}`,
-      );
-      // Mark done so a re-run doesn't keep paying the classifier cost on
-      // sessions the model can't extract anything from.
-      done.add(sid);
-      if (!opts.dryRun && processed % SAVE_EVERY === 0) saveState(statePath, done);
-      continue;
-    }
-
-    if (opts.dryRun) {
-      factsWritten += facts.length;
-      processed += 1;
-      opts.onProgress?.(i + 1, total, sid, "ok", `would-write=${facts.length}`);
-      continue;
-    }
-
-    try {
-      await opts.store.insertFactsForSession(
-        sid,
-        opts.factStore,
-        facts,
-        opts.embedder ?? null,
-      );
-    } catch (err) {
-      storageFailures += 1;
-      const detail = err instanceof Error ? err.message : String(err);
-      opts.onProgress?.(i + 1, total, sid, "storage_failed", detail);
-      continue;
-    }
-
-    factsWritten += facts.length;
-    processed += 1;
-    done.add(sid);
-    opts.onProgress?.(i + 1, total, sid, "ok", `wrote=${facts.length}`);
-    if (processed % SAVE_EVERY === 0) saveState(statePath, done);
-  }
-
-  if (!opts.dryRun) saveState(statePath, done);
-
-  return {
-    total,
-    processed,
-    factsWritten,
-    skippedAlreadyDone,
-    skippedExistingFacts,
-    skippedNoBody,
-    skippedLowConfidence,
-    classifyFailures,
-    storageFailures,
-  };
-}