nlm-memory 0.4.0

package/scripts/longmemeval/scorer.ts

@@ -0,0 +1,99 @@
+/**
+ * Pure scoring functions for the LongMemEval harness. Two metrics:
+ *
+ *  - R@k (recall at k): did the retriever return any gold session ID in
+ *    its top-k results? Standard benchmark metric.
+ *  - Session-body hit: did the gold answer text appear anywhere in the
+ *    bodies of the top-k returned sessions? NLM-specific companion that
+ *    captures session-as-primary-unit value the strict-ID R@k can miss
+ *    (e.g. a session that supersedes the gold session and quotes its
+ *    decision).
+ *
+ * Both functions are deterministic and dependency-free so the harness can
+ * test them with synthetic inputs.
+ */
+
+export interface ScoreInputs {
+  readonly returnedIds: ReadonlyArray<string>;
+  readonly goldIds: ReadonlyArray<string>;
+  /** Map id → body for the bodies of the top-k returned sessions. */
+  readonly returnedBodies: ReadonlyArray<string>;
+  /** Some LongMemEval answers are ints (counting questions); coerced to string. */
+  readonly answer: string | number | boolean;
+  readonly k: number;
+}
+
+export interface SingleScore {
+  readonly recallAtK: 0 | 1;
+  readonly sessionBodyHit: 0 | 1;
+}
+
+/** Score a single question. Returns 0/1 indicators that aggregate via mean. */
+export function scoreOne(input: ScoreInputs): SingleScore {
+  const topK = input.returnedIds.slice(0, input.k);
+  const goldSet = new Set(input.goldIds);
+  const recallAtK = topK.some((id) => goldSet.has(id)) ? 1 : 0;
+
+  // Session-body hit: substring match for multi-word answers; word-boundary
+  // match for short answers (single token <4 chars: "3", "yes", numeric
+  // counts). Without the boundary, a numeric answer "3" hits every body
+  // containing "3 days", "$3", etc., inflating the metric to noise.
+  const ans = normalize(String(input.answer));
+  let sessionBodyHit: 0 | 1 = 0;
+  if (ans.length > 0) {
+    const isShortToken = !ans.includes(" ") && ans.length < 4;
+    const test = isShortToken
+      ? (body: string): boolean =>
+          new RegExp(`\\b${escapeRegExp(ans)}\\b`).test(normalize(body))
+      : (body: string): boolean => normalize(body).includes(ans);
+    const bodies = input.returnedBodies.slice(0, input.k);
+    for (const body of bodies) {
+      if (test(body)) {
+        sessionBodyHit = 1;
+        break;
+      }
+    }
+  }
+  return { recallAtK, sessionBodyHit };
+}
+
+export interface Aggregate {
+  readonly n: number;
+  readonly recallAtK: number;
+  readonly sessionBodyHitRate: number;
+}
+
+/** Aggregate per-question scores into mean rates. */
+export function aggregate(scores: ReadonlyArray<SingleScore>): Aggregate {
+  const n = scores.length;
+  if (n === 0) {
+    return { n: 0, recallAtK: 0, sessionBodyHitRate: 0 };
+  }
+  let r = 0;
+  let s = 0;
+  for (const x of scores) {
+    r += x.recallAtK;
+    s += x.sessionBodyHit;
+  }
+  return {
+    n,
+    recallAtK: round3(r / n),
+    sessionBodyHitRate: round3(s / n),
+  };
+}
+
+function normalize(s: string): string {
+  return s
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]/gu, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function round3(x: number): number {
+  return Math.round(x * 1000) / 1000;
+}
+
+function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}

package/scripts/longmemeval/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "../..",
+    "outDir": "../../dist-scripts",
+    "noEmit": true
+  },
+  "include": ["./*.ts", "../../src/**/*.ts"]
+}

package/scripts/longmemeval/types.ts

@@ -0,0 +1,35 @@
+/**
+ * LongMemEval dataset schema. Mirrors the published JSON shape from
+ * huggingface.co/datasets/xiaowu0162/longmemeval-cleaned.
+ *
+ * Each instance: a question against a haystack of past chat sessions. The
+ * gold session IDs are in `answer_session_ids` — that's what the retrieval
+ * step is scored against (R@k: was any gold ID returned in the top k).
+ */
+
+export interface LongMemEvalTurn {
+  readonly role: "user" | "assistant";
+  readonly content: string;
+  readonly has_answer?: boolean;
+}
+
+export interface LongMemEvalInstance {
+  readonly question_id: string;
+  readonly question_type: string;
+  readonly question: string;
+  // LongMemEval answers are sometimes ints/booleans for counting and
+  // temporal-reasoning questions — coerce at the call site.
+  readonly answer: string | number | boolean;
+  readonly question_date: string;
+  readonly haystack_session_ids: ReadonlyArray<string>;
+  readonly haystack_dates: ReadonlyArray<string>;
+  readonly haystack_sessions: ReadonlyArray<ReadonlyArray<LongMemEvalTurn>>;
+  readonly answer_session_ids: ReadonlyArray<string>;
+}
+
+/** Serialize a session's turn list to a single body string for NLM ingest. */
+export function turnsToBody(turns: ReadonlyArray<LongMemEvalTurn>): string {
+  return turns
+    .map((t) => `${t.role === "user" ? "User" : "Assistant"}: ${t.content}`)
+    .join("\n\n");
+}

package/scripts/nlm-daily-digest.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+"""NLM daily digest — composes and posts a Telegram summary of recall activity.
+
+Called from nlm-daily-digest.sh. Reads from the local NLM daemon (default
+http://localhost:3940) and posts to the Telegram chat configured in env.
+
+Token / chat id are read from the Whtnxt Agent .env (sourced by the wrapper
+shell script). NLM port defaults to 3940 but honors NLM_PORT.
+
+No external deps — urllib only. Exits non-zero on any unrecoverable error so
+the cron log captures the failure.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import urllib.parse
+import urllib.request
+from collections import Counter
+from datetime import datetime, time, timedelta, timezone
+from pathlib import Path
+from typing import Any
+
+# Hook log lives alongside the daemon DB at ~/.nlm/hook-log.jsonl unless
+# overridden via NLM_HOOK_LOG (mirrors the TS hook-log module).
+HOOK_LOG_PATH = Path(os.environ.get("NLM_HOOK_LOG", str(Path.home() / ".nlm" / "hook-log.jsonl")))
+
+# Local timezone for yesterday-window math. Cron fires at 7am CT and Edward
+# thinks in CT — "yesterday" means yesterday-in-CT, not yesterday-in-UTC.
+LOCAL_TZ = datetime.now().astimezone().tzinfo
+
+# Patterns that mark a recall as a probe/test, not real agent usage.
+# Match is case-insensitive substring against the query text.
+PROBE_PATTERNS: tuple[str, ...] = (
+    "concurrency probe",
+    "test probe",
+    "path test",
+    "recall test",
+    "smoke",
+    "cutover-test",
+)
+
+
+def is_probe(query: str | None) -> bool:
+    if not query:
+        return False
+    q = query.lower()
+    return any(p in q for p in PROBE_PATTERNS)
+
+
+def http_get_json(url: str, timeout: int = 5) -> Any:
+    req = urllib.request.Request(url, headers={"User-Agent": "nlm-daily-digest/1.0"})
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        return json.loads(resp.read().decode("utf-8"))
+
+
+def post_telegram(token: str, chat_id: str, text: str) -> None:
+    url = f"https://api.telegram.org/bot{token}/sendMessage"
+    payload = urllib.parse.urlencode({
+        "chat_id": chat_id,
+        "text": text,
+        "disable_web_page_preview": "true",
+    }).encode("utf-8")
+    req = urllib.request.Request(
+        url,
+        data=payload,
+        headers={"User-Agent": "nlm-daily-digest/1.0"},
+    )
+    with urllib.request.urlopen(req, timeout=10) as resp:
+        body = json.loads(resp.read().decode("utf-8"))
+        if not body.get("ok"):
+            raise RuntimeError(f"telegram api error: {body}")
+
+
+def yesterday_window() -> tuple[datetime, datetime]:
+    """Return (start, end) of yesterday in local tz, both tz-aware."""
+    today_local = datetime.now(LOCAL_TZ).date()
+    yesterday = today_local - timedelta(days=1)
+    start = datetime.combine(yesterday, time(0, 0), tzinfo=LOCAL_TZ)
+    end = datetime.combine(today_local, time(0, 0), tzinfo=LOCAL_TZ)
+    return start, end
+
+
+def parse_iso(ts: str) -> datetime | None:
+    try:
+        return datetime.fromisoformat(ts.replace("Z", "+00:00"))
+    except (TypeError, ValueError, AttributeError):
+        return None
+
+
+def hook_liveness_check(dataset_sessions: list[dict[str, Any]]) -> str | None:
+    """Return an alert string if Claude Code ran yesterday but the hook did not fire.
+
+    Returns None when either: (a) no Claude Code sessions yesterday, in which
+    case silence is expected, or (b) hook fires were recorded as expected.
+
+    This is the load-bearing liveness check. The install-time smoke test only
+    catches malformed commands at install moment — this catches post-install
+    drift (node upgrades, dist moves, settings.json hand-edits, Claude Code
+    hook dispatcher changes) by correlating real usage with real hook output.
+    """
+    start, end = yesterday_window()
+
+    cc_sessions_yesterday = 0
+    for s in dataset_sessions:
+        if not str(s.get("runtime", "")).startswith("claude-code"):
+            continue
+        ts = parse_iso(str(s.get("started_at", "")))
+        if ts is None:
+            continue
+        if start <= ts < end:
+            cc_sessions_yesterday += 1
+
+    if cc_sessions_yesterday == 0:
+        return None  # Edward didn't use Claude Code yesterday; silence is fine.
+
+    if not HOOK_LOG_PATH.exists():
+        return (
+            f"⚠️ hook silent: {cc_sessions_yesterday} Claude Code sessions yesterday, "
+            f"0 hook fires (log file missing at {HOOK_LOG_PATH})"
+        )
+
+    live_fires_yesterday = 0
+    with HOOK_LOG_PATH.open("r", encoding="utf-8") as f:
+        for line in f:
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if entry.get("mode") != "live":
+                continue
+            ts = parse_iso(str(entry.get("ts", "")))
+            if ts is None:
+                continue
+            if start <= ts < end:
+                live_fires_yesterday += 1
+
+    if live_fires_yesterday == 0:
+        return (
+            f"⚠️ hook silent: {cc_sessions_yesterday} Claude Code sessions yesterday, "
+            f"0 live hook fires — check `nlm hook install` + ~/.claude/settings.json"
+        )
+    return None
+
+
+def compose(stats: dict[str, Any], recent: list[dict[str, Any]], port: int, hook_alert: str | None = None) -> str:
+    """Format the digest message body."""
+    # `stats` window is server-side (7 days currently). `recent` is the
+    # last ~200 events we use to compute the *24h* real-traffic slice.
+    cutoff = datetime.now(timezone.utc) - timedelta(hours=24)
+    real_24h: list[dict[str, Any]] = []
+    for e in recent:
+        try:
+            ts = datetime.fromisoformat(e["ts"].replace("Z", "+00:00"))
+        except (KeyError, ValueError):
+            continue
+        if ts < cutoff:
+            continue
+        if is_probe(e.get("query")):
+            continue
+        real_24h.append(e)
+
+    src_24h = Counter(e.get("source", "?") for e in real_24h)
+    top_q = Counter(e.get("query", "") for e in real_24h if e.get("query")).most_common(5)
+
+    # 7-day stats (server-computed) — strip probes from the totals.
+    total_7d = int(stats.get("total", 0))
+    probes_7d = sum(
+        count for q, count in stats.get("top_queries", [])
+        if is_probe(q)
+    ) if isinstance(stats.get("top_queries"), list) else 0
+    real_7d = max(total_7d - probes_7d, 0)
+
+    useful = stats.get("useful_hit_rate")
+    useful_line = (
+        "useful_hit_rate: pending (see docs/methodology/useful-hit-rate.md)"
+        if useful is None
+        else f"useful_hit_rate (7d): {useful:.0%}"
+    )
+
+    src_str = " · ".join(f"{k}={v}" for k, v in sorted(src_24h.items())) or "none"
+
+    top_lines = "\n".join(f"  {n}. {q[:60]}" for n, (q, _) in enumerate(top_q, 1)) or "  (none)"
+
+    today = datetime.now().strftime("%a %Y-%m-%d")
+    alert_block = f"{hook_alert}\n\n" if hook_alert else ""
+    return (
+        f"NLM digest — {today}\n"
+        f"\n"
+        f"{alert_block}"
+        f"Last 24h (real traffic): {len(real_24h)} queries · {src_str}\n"
+        f"Last 7d: {real_7d} real / {total_7d} total · hit_rate {float(stats.get('hit_rate', 0)):.0%}\n"
+        f"{useful_line}\n"
+        f"\n"
+        f"Top real queries (24h):\n"
+        f"{top_lines}\n"
+        f"\n"
+        f"UI: http://localhost:{port}/ui/"
+    )
+
+
+def main() -> int:
+    port = int(os.environ.get("NLM_PORT", "3940"))
+    token = os.environ.get("TELEGRAM_BOT_TOKEN")
+    chat_id = os.environ.get("TELEGRAM_CHAT_ID")
+    if not token or not chat_id:
+        print("missing TELEGRAM_BOT_TOKEN or TELEGRAM_CHAT_ID", file=sys.stderr)
+        return 2
+
+    base = f"http://localhost:{port}"
+    try:
+        stats = http_get_json(f"{base}/api/recall/stats")
+        recent_resp = http_get_json(f"{base}/api/recall/recent?limit=200")
+        dataset_resp = http_get_json(f"{base}/api/dataset", timeout=15)
+    except Exception as e:
+        # Daemon down — post an alert instead of failing silently.
+        text = f"NLM digest — {datetime.now().strftime('%a %Y-%m-%d')}\n\nDaemon unreachable at {base}\n{e}"
+        try:
+            post_telegram(token, chat_id, text)
+        except Exception as send_err:
+            print(f"telegram send also failed: {send_err}", file=sys.stderr)
+        return 1
+
+    recent: list[dict[str, Any]] = recent_resp.get("entries", []) if isinstance(recent_resp, dict) else []
+    dataset_sessions: list[dict[str, Any]] = (
+        dataset_resp.get("sessions", []) if isinstance(dataset_resp, dict) else []
+    )
+    hook_alert = hook_liveness_check(dataset_sessions)
+    text = compose(stats, recent, port, hook_alert=hook_alert)
+    post_telegram(token, chat_id, text)
+    print(f"[{datetime.now().isoformat()}] digest posted ({len(text)} chars)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/nlm-daily-digest.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# NLM daily digest — posts a morning Telegram summary of recall activity.
+#
+# Cron entry (7:00am CT, runs after daily-reminders at 6:50am):
+#   0 7 * * * "/Users/echalupa/Documents/Coding Projects/nlm-memory-ts/scripts/nlm-daily-digest.sh" >> "/Users/echalupa/Documents/Coding Projects/nlm-memory-ts/logs/daily-digest/digest.log" 2>&1
+#
+# Reads credentials from the Whtnxt Agent .env (Telegram bot token + chat id).
+
+set -euo pipefail
+
+REPO_DIR="/Users/echalupa/Documents/Coding Projects/nlm-memory-ts"
+WHTNXT_ENV="/Users/echalupa/Documents/Coding Projects/Whtnxt Agent/.env"
+LOG_DIR="${REPO_DIR}/logs/daily-digest"
+mkdir -p "${LOG_DIR}"
+
+# shellcheck source=/dev/null
+set -a
+. "${WHTNXT_ENV}"
+set +a
+
+: "${TELEGRAM_BOT_TOKEN:?TELEGRAM_BOT_TOKEN missing}"
+: "${TELEGRAM_CHAT_ID:?TELEGRAM_CHAT_ID missing}"
+
+# Populate useful-hit-log before stats fetch so useful_hit_rate is live.
+/Users/echalupa/.nvm/versions/node/v22.22.1/bin/nlm useful-scan --days 1 \
+  >> "${LOG_DIR}/useful-scan.log" 2>&1 || true
+
+python3 "${REPO_DIR}/scripts/nlm-daily-digest.py"

package/src/cli/classify-parity.ts

@@ -0,0 +1,257 @@
+/**
+ * `nlm classify-parity` — Phase C parity verification harness.
+ *
+ * Reads N sessions from ~/.nlm/canonical.sqlite (read-only by default),
+ * runs the TS OllamaClient.classify on each body, diffs the result
+ * against the persisted Python classifier output, and prints aggregate
+ * metrics: Jaccard similarity on entities/decisions/open sets, label
+ * exact match rate, summary length delta, schema-failure count.
+ *
+ * Safe: opens the live store in readonly mode. Does not write anything
+ * back. Designed to be run interactively from a terminal during the
+ * Phase C cutover-prep window.
+ */
+
+import { homedir } from "node:os";
+import { resolve } from "node:path";
+import Database from "better-sqlite3";
+import * as sqliteVec from "sqlite-vec";
+import type { LLMClient } from "../ports/llm-client.js";
+import { LLMUnreachableError } from "../ports/llm-client.js";
+import { DeepSeekClient } from "../llm/deepseek-client.js";
+import { OllamaClient, ClassifierSchemaError } from "../llm/ollama-client.js";
+import { autoloadEnv } from "../llm/env-autoload.js";
+
+export type Provider = "ollama" | "deepseek";
+
+interface CliOptions {
+  readonly limit: number;
+  readonly dbPath: string;
+  readonly ollamaUrl: string;
+  readonly classifyModel: string;
+  readonly provider: Provider;
+  readonly verbose: boolean;
+}
+
+interface SessionRow {
+  id: string;
+  label: string;
+  summary: string;
+  body: string | null;
+}
+
+interface PersistedClassification {
+  label: string;
+  summary: string;
+  entities: string[];
+  decisions: string[];
+  open: string[];
+}
+
+interface DiffMetrics {
+  sessionId: string;
+  labelMatch: boolean;
+  labelTs: string;
+  labelPy: string;
+  entityJaccard: number;
+  decisionJaccard: number;
+  openJaccard: number;
+  summaryDeltaChars: number;
+  schemaFailure: boolean;
+  errorMessage?: string;
+}
+
+export interface ParityReport {
+  attempted: number;
+  succeeded: number;
+  schemaFailures: number;
+  networkFailures: number;
+  labelExactMatchRate: number;
+  meanEntityJaccard: number;
+  meanDecisionJaccard: number;
+  meanOpenJaccard: number;
+  diffs: ReadonlyArray<DiffMetrics>;
+}
+
+function parseArgs(argv: string[]): CliOptions {
+  const flag = (name: string, fallback?: string): string | undefined => {
+    const i = argv.indexOf(name);
+    if (i === -1) return fallback;
+    return argv[i + 1] ?? fallback;
+  };
+  const limit = Number.parseInt(flag("--limit", "10") ?? "10", 10);
+  const providerRaw = (flag("--provider", "deepseek") ?? "deepseek").toLowerCase();
+  const provider: Provider = providerRaw === "ollama" ? "ollama" : "deepseek";
+  const defaultModel = provider === "deepseek" ? "deepseek-v4-flash" : "phi4-mini:latest";
+  return {
+    limit: Number.isFinite(limit) && limit > 0 ? limit : 10,
+    dbPath:
+      flag("--db", process.env["NLM_DB_PATH"] ?? resolve(homedir(), ".nlm/canonical.sqlite")) ??
+      resolve(homedir(), ".nlm/canonical.sqlite"),
+    ollamaUrl: flag("--ollama", process.env["NLM_OLLAMA_URL"] ?? "http://localhost:11434") ?? "http://localhost:11434",
+    classifyModel: flag("--model", defaultModel) ?? defaultModel,
+    provider,
+    verbose: argv.includes("--verbose"),
+  };
+}
+
+function buildClient(opts: { provider: Provider; classifyModel: string; ollamaUrl: string }): LLMClient {
+  if (opts.provider === "deepseek") {
+    autoloadEnv();
+    return new DeepSeekClient({ classifyModel: opts.classifyModel });
+  }
+  return new OllamaClient({ baseUrl: opts.ollamaUrl, classifyModel: opts.classifyModel });
+}
+
+function jaccard(a: ReadonlyArray<string>, b: ReadonlyArray<string>): number {
+  const setA = new Set(a.map((s) => s.toLowerCase().trim()));
+  const setB = new Set(b.map((s) => s.toLowerCase().trim()));
+  if (setA.size === 0 && setB.size === 0) return 1;
+  const inter = new Set([...setA].filter((x) => setB.has(x)));
+  const union = new Set([...setA, ...setB]);
+  return inter.size / union.size;
+}
+
+export async function runParity(opts: CliOptions): Promise<ParityReport> {
+  const db = new Database(opts.dbPath, { readonly: true });
+  sqliteVec.load(db);
+
+  const rows = db
+    .prepare<[number], SessionRow>(
+      `SELECT id, label, summary, body
+       FROM sessions
+       WHERE body IS NOT NULL AND body != ''
+       ORDER BY started_at DESC
+       LIMIT ?`,
+    )
+    .all(opts.limit);
+
+  const persistedById = new Map<string, PersistedClassification>();
+  for (const r of rows) {
+    const entities = db
+      .prepare<[string], { entity_canonical: string }>(
+        "SELECT entity_canonical FROM session_entities WHERE session_id = ?",
+      )
+      .all(r.id)
+      .map((x) => x.entity_canonical);
+    const markers = db
+      .prepare<[string], { kind: "decision" | "open"; text: string }>(
+        "SELECT kind, text FROM markers WHERE session_id = ? ORDER BY position",
+      )
+      .all(r.id);
+    persistedById.set(r.id, {
+      label: r.label,
+      summary: r.summary,
+      entities,
+      decisions: markers.filter((m) => m.kind === "decision").map((m) => m.text),
+      open: markers.filter((m) => m.kind === "open").map((m) => m.text),
+    });
+  }
+  db.close();
+
+  const client = buildClient(opts);
+
+  const diffs: DiffMetrics[] = [];
+  let schemaFailures = 0;
+  let networkFailures = 0;
+
+  let idx = 0;
+  for (const r of rows) {
+    idx += 1;
+    const py = persistedById.get(r.id);
+    if (!py || !r.body) continue;
+
+    const t0 = Date.now();
+    try {
+      const ts = await client.classify(r.body);
+      const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
+      const labelMatch = ts.label.toLowerCase().trim() === py.label.toLowerCase().trim();
+      const entJ = jaccard(ts.entities, py.entities);
+      const decJ = jaccard(ts.decisions, py.decisions);
+      const openJ = jaccard(ts.open, py.open);
+      diffs.push({
+        sessionId: r.id,
+        labelMatch,
+        labelTs: ts.label,
+        labelPy: py.label,
+        entityJaccard: entJ,
+        decisionJaccard: decJ,
+        openJaccard: openJ,
+        summaryDeltaChars: ts.summary.length - py.summary.length,
+        schemaFailure: false,
+      });
+      if (opts.verbose) {
+        const tag = labelMatch ? "EQ " : "DIFF";
+        process.stderr.write(
+          `  [${idx}/${rows.length}] ${elapsed}s ${tag} ${r.id}  ent=${entJ.toFixed(2)} dec=${decJ.toFixed(2)} open=${openJ.toFixed(2)}\n`,
+        );
+      }
+    } catch (e) {
+      const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
+      const message = e instanceof Error ? e.message : String(e);
+      if (e instanceof ClassifierSchemaError) schemaFailures += 1;
+      else if (e instanceof LLMUnreachableError) networkFailures += 1;
+      diffs.push({
+        sessionId: r.id,
+        labelMatch: false,
+        labelTs: "",
+        labelPy: py.label,
+        entityJaccard: 0,
+        decisionJaccard: 0,
+        openJaccard: 0,
+        summaryDeltaChars: 0,
+        schemaFailure: e instanceof ClassifierSchemaError,
+        errorMessage: message,
+      });
+      if (opts.verbose) {
+        process.stderr.write(
+          `  [${idx}/${rows.length}] ${elapsed}s ERR  ${r.id}  :: ${message}\n`,
+        );
+      }
+    }
+  }
+
+  const successes = diffs.filter((d) => !d.errorMessage);
+  const mean = (xs: ReadonlyArray<number>): number =>
+    xs.length === 0 ? 0 : Math.round((xs.reduce((a, b) => a + b, 0) / xs.length) * 1000) / 1000;
+
+  return {
+    attempted: diffs.length,
+    succeeded: successes.length,
+    schemaFailures,
+    networkFailures,
+    labelExactMatchRate: mean(successes.map((d) => (d.labelMatch ? 1 : 0))),
+    meanEntityJaccard: mean(successes.map((d) => d.entityJaccard)),
+    meanDecisionJaccard: mean(successes.map((d) => d.decisionJaccard)),
+    meanOpenJaccard: mean(successes.map((d) => d.openJaccard)),
+    diffs,
+  };
+}
+
+export async function main(): Promise<void> {
+  const opts = parseArgs(process.argv.slice(2));
+  console.error(`nlm classify-parity: ${opts.limit} sessions from ${opts.dbPath}`);
+  console.error(
+    `  provider: ${opts.provider}  model: ${opts.classifyModel}` +
+      (opts.provider === "ollama" ? `  ollama: ${opts.ollamaUrl}` : ""),
+  );
+  const report = await runParity(opts);
+
+  console.error("");
+  console.error(`attempted:           ${report.attempted}`);
+  console.error(`succeeded:           ${report.succeeded}`);
+  console.error(`schema failures:     ${report.schemaFailures}`);
+  console.error(`network failures:    ${report.networkFailures}`);
+  console.error(`label exact match:   ${(report.labelExactMatchRate * 100).toFixed(1)}%`);
+  console.error(`mean Jaccard ents:   ${report.meanEntityJaccard.toFixed(3)}`);
+  console.error(`mean Jaccard decs:   ${report.meanDecisionJaccard.toFixed(3)}`);
+  console.error(`mean Jaccard open:   ${report.meanOpenJaccard.toFixed(3)}`);
+}
+
+const isMain = import.meta.url === `file://${process.argv[1]}`;
+if (isMain) {
+  main().catch((e) => {
+    console.error("classify-parity fatal:", e);
+    process.exit(1);
+  });
+}