nlm-memory 0.4.2 → 0.5.1

package/src/core/recall/citation-log.ts

@@ -1,108 +0,0 @@
-/**
- * Append-only JSONL citation log. One line per (conversationId, citedId)
- * that the Stop hook detected. This is the training-data substrate for the
- * future learned reranker: each row is a (query, returned_id, was_cited)
- * triple once joined against ~/.nlm/query_log.jsonl by conversationId.
- *
- * Path defaults to ~/.nlm/citation-log.jsonl, overridable via
- * NLM_CITATION_LOG. Telemetry path — never raises.
- */
-
-import { appendFile, mkdir, readFile, stat } from "node:fs/promises";
-import { dirname, join } from "node:path";
-import { homedir } from "node:os";
-
-export type CitationKind = "tool_use" | "prose";
-
-export interface CitationEntry {
-  readonly conversationId: string;
-  readonly citedId: string;
-  readonly kind?: CitationKind;
-  readonly responsePreview?: string;
-}
-
-export interface CitationStats {
-  readonly days: number;
-  readonly total: number;
-  readonly distinct_ids: number;
-  readonly top_ids: ReadonlyArray<{ readonly id: string; readonly count: number }>;
-  readonly log_present: boolean;
-}
-
-function defaultLogPath(): string {
-  return process.env["NLM_CITATION_LOG"] ?? join(homedir(), ".nlm", "citation-log.jsonl");
-}
-
-export async function appendCitation(
-  entry: CitationEntry,
-  logPath: string = defaultLogPath(),
-): Promise<void> {
-  try {
-    await mkdir(dirname(logPath), { recursive: true });
-    const payload = {
-      ts: new Date().toISOString(),
-      conversation_id: entry.conversationId,
-      cited_id: entry.citedId,
-      ...(entry.kind !== undefined ? { kind: entry.kind } : {}),
-      ...(entry.responsePreview !== undefined
-        ? { response_preview: entry.responsePreview }
-        : {}),
-    };
-    await appendFile(logPath, JSON.stringify(payload) + "\n", "utf8");
-  } catch {
-    // Telemetry failure must never break the call path.
-  }
-}
-
-export async function citationStats(
-  days: number,
-  logPath: string = defaultLogPath(),
-): Promise<CitationStats> {
-  const base: CitationStats = {
-    days,
-    total: 0,
-    distinct_ids: 0,
-    top_ids: [],
-    log_present: false,
-  };
-  try {
-    await stat(logPath);
-  } catch {
-    return base;
-  }
-  let raw: string;
-  try {
-    raw = await readFile(logPath, "utf8");
-  } catch {
-    return { ...base, log_present: true };
-  }
-  const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
-  const counts = new Map<string, number>();
-  let total = 0;
-  for (const line of raw.split("\n")) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-    let entry: Record<string, unknown>;
-    try {
-      entry = JSON.parse(trimmed) as Record<string, unknown>;
-    } catch {
-      continue;
-    }
-    const tsRaw = entry["ts"];
-    if (typeof tsRaw !== "string") continue;
-    const ts = Date.parse(tsRaw);
-    if (!Number.isFinite(ts) || ts < cutoff) continue;
-    const id = entry["cited_id"];
-    if (typeof id !== "string" || !id) continue;
-    total += 1;
-    counts.set(id, (counts.get(id) ?? 0) + 1);
-  }
-  const sorted = [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 20);
-  return {
-    days,
-    total,
-    distinct_ids: counts.size,
-    top_ids: sorted.map(([id, count]) => ({ id, count })),
-    log_present: true,
-  };
-}

package/src/core/recall/filter.ts

@@ -1,27 +0,0 @@
-/**
- * Session-list filters used before scoring.
- *
- * Pure function over a session array. Mirrors recall.py:_apply_filters.
- */
-
-import type { Session, RecallKindFilter } from "@shared/types.js";
-
-export interface RecallFilter {
-  readonly entity?: string;
-  readonly kind?: RecallKindFilter;
-}
-
-export function applyFilter(
-  sessions: ReadonlyArray<Session>,
-  filter: RecallFilter,
-): ReadonlyArray<Session> {
-  const { entity, kind } = filter;
-  if (!entity && !kind) return sessions;
-
-  return sessions.filter((s) => {
-    if (entity && !s.entities.includes(entity)) return false;
-    if (kind === "decision" && s.decisions.length === 0) return false;
-    if (kind === "open" && s.open.length === 0) return false;
-    return true;
-  });
-}

package/src/core/recall/index.ts

@@ -1,6 +0,0 @@
-export { RecallService } from "./recall-service.js";
-export type { RecallServiceDeps } from "./recall-service.js";
-export { keywordMatchFields } from "./match-fields.js";
-export { applyFilter } from "./filter.js";
-export type { RecallFilter } from "./filter.js";
-export { tokenize, tokenSet } from "./tokenize.js";

package/src/core/recall/match-fields.ts

@@ -1,40 +0,0 @@
-/**
- * Computes which session fields a keyword query matched, for the `matchedIn`
- * badge on a RecallHit. Pure function — no DB, no I/O. FTS5 BM25 ranks the
- * whole row; this recovers per-field attribution from the resolved Session,
- * including decisions/open which live in the markers table (not in FTS).
- */
-
-import type { MatchField, Session } from "@shared/types.js";
-import { tokenSet } from "./tokenize.js";
-
-type SessionFields = Pick<Session, "label" | "summary" | "decisions" | "open">;
-
-export function keywordMatchFields(
-  session: SessionFields,
-  queryTokens: ReadonlySet<string>,
-): ReadonlyArray<MatchField> {
-  if (queryTokens.size === 0) return [];
-  const fields: MatchField[] = [];
-
-  if (overlaps(queryTokens, tokenSet(session.label))) fields.push("label");
-  if (overlaps(queryTokens, joinedTokens(session.decisions))) fields.push("decisions");
-  if (overlaps(queryTokens, joinedTokens(session.open))) fields.push("open");
-  if (overlaps(queryTokens, tokenSet(session.summary))) fields.push("summary");
-
-  return fields;
-}
-
-function joinedTokens(values: ReadonlyArray<string>): Set<string> {
-  const out = new Set<string>();
-  for (const v of values) {
-    for (const t of tokenSet(v)) out.add(t);
-  }
-  return out;
-}
-
-function overlaps(a: ReadonlySet<string>, b: ReadonlySet<string>): boolean {
-  const [small, large] = a.size <= b.size ? [a, b] : [b, a];
-  for (const item of small) if (large.has(item)) return true;
-  return false;
-}

package/src/core/recall/query-log.ts

@@ -1,149 +0,0 @@
-/**
- * Query log + stats aggregation. Mirrors recall.py's log_query() / stats().
- *
- * Telemetry path — never raises. The HTTP recall handler calls logQuery()
- * after each /api/recall response; /api/recall/stats reads the same file
- * back to drive the Pulse agent-recall observability panel.
- *
- * File format: one JSON object per line at $NLM_QUERY_LOG or
- * ~/.nlm/query_log.jsonl. Append-only.
- */
-
-import { appendFile, mkdir, readFile, stat } from "node:fs/promises";
-import { dirname, join } from "node:path";
-import { homedir } from "node:os";
-import type { RecallKindFilter, RecallMode } from "@shared/types.js";
-import { readUsefulHitRate, defaultUsefulHitLogPath } from "./useful-scan.js";
-
-export interface LogEntry {
-  readonly source: string;
-  readonly query: string | null;
-  readonly entity: string | null;
-  readonly kind: RecallKindFilter | null;
-  readonly mode: RecallMode;
-  readonly limit: number;
-  readonly nResults: number;
-  readonly returnedIds: ReadonlyArray<string>;
-}
-
-export interface StatsResult {
-  readonly days: number;
-  readonly total: number;
-  readonly with_results: number;
-  readonly hit_rate: number;
-  // null until nlm useful-scan has run and populated useful-hit-log.jsonl with
-  // measurable entries. Non-null once ≥1 measurable entry exists in the window.
-  readonly useful_hit_rate: number | null;
-  readonly by_source: Record<string, number>;
-  readonly top_queries: ReadonlyArray<{ readonly query: string; readonly count: number }>;
-  readonly log_present: boolean;
-}
-
-function defaultLogPath(): string {
-  return process.env["NLM_QUERY_LOG"] ?? join(homedir(), ".nlm", "query_log.jsonl");
-}
-
-export async function logQuery(
-  entry: LogEntry,
-  logPath: string = defaultLogPath(),
-): Promise<void> {
-  try {
-    await mkdir(dirname(logPath), { recursive: true });
-    const payload = {
-      ts: new Date().toISOString(),
-      source: entry.source,
-      query: entry.query,
-      entity: entry.entity,
-      kind: entry.kind,
-      mode: entry.mode,
-      limit: entry.limit,
-      n_results: entry.nResults,
-      returned_ids: entry.returnedIds,
-    };
-    await appendFile(logPath, JSON.stringify(payload) + "\n", "utf8");
-  } catch {
-    // Telemetry must never break the call path.
-  }
-}
-
-export async function recallStats(
-  days: number,
-  logPath: string = defaultLogPath(),
-  usefulHitLogPath: string = defaultUsefulHitLogPath(),
-): Promise<StatsResult> {
-  const useful_hit_rate = await readUsefulHitRate(usefulHitLogPath, days);
-
-  const base: StatsResult = {
-    days,
-    total: 0,
-    with_results: 0,
-    hit_rate: 0,
-    useful_hit_rate,
-    by_source: {},
-    top_queries: [],
-    log_present: false,
-  };
-
-  try {
-    await stat(logPath);
-  } catch {
-    return base;
-  }
-
-  const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
-  const bySource = new Map<string, number>();
-  const queryCounts = new Map<string, number>();
-  let total = 0;
-  let withResults = 0;
-
-  let raw: string;
-  try {
-    raw = await readFile(logPath, "utf8");
-  } catch {
-    return { ...base, log_present: true };
-  }
-
-  for (const line of raw.split("\n")) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-    let entry: Record<string, unknown>;
-    try {
-      entry = JSON.parse(trimmed) as Record<string, unknown>;
-    } catch {
-      continue;
-    }
-    const tsRaw = entry["ts"];
-    if (typeof tsRaw !== "string") continue;
-    const ts = Date.parse(tsRaw);
-    if (!Number.isFinite(ts) || ts < cutoff) continue;
-
-    total += 1;
-    const n = typeof entry["n_results"] === "number" ? entry["n_results"] : 0;
-    if (n > 0) withResults += 1;
-
-    const source = typeof entry["source"] === "string" ? entry["source"] : "unknown";
-    bySource.set(source, (bySource.get(source) ?? 0) + 1);
-
-    const q = entry["query"];
-    if (typeof q === "string" && q) {
-      const norm = q.toLowerCase().trim();
-      queryCounts.set(norm, (queryCounts.get(norm) ?? 0) + 1);
-    }
-  }
-
-  const sortedSources = [...bySource.entries()].sort((a, b) => b[1] - a[1]);
-  const sortedQueries = [...queryCounts.entries()]
-    .sort((a, b) => b[1] - a[1])
-    .slice(0, 5);
-
-  return {
-    days,
-    total,
-    with_results: withResults,
-    hit_rate: total === 0 ? 0 : Math.round((withResults / total) * 1000) / 1000,
-    useful_hit_rate,
-    by_source: Object.fromEntries(sortedSources),
-    top_queries: sortedQueries.map(([query, count]) => ({ query, count })),
-    log_present: true,
-  };
-}

package/src/core/recall/query-shape.ts

@@ -1,66 +0,0 @@
-/**
- * Query shape detection for force-include of keyword rank-1 in hybrid recall.
- *
- * Build F (2026-05-26): when a query has both a temporal marker and a
- * named-entity-shaped token, the keyword-leg rank-1 session is force-included
- * in the merged top-k result. Diagnostic justification: of 7 hybrid temporal
- * misses where keyword found the right session, 5 had keyword rank=1 and
- * pure RRF demoted them out of top-5 because the same session wasn't in
- * semantic's top-15. Build E′ (asymmetric multiplicative boost) contributed
- * zero — boost magnitude was too small to overcome the "appears in both lists
- * at lower rank" advantage. Force-include sidesteps RRF math entirely.
- *
- * Probe data (n=500 LongMemEval-S, hybrid k=5):
- *   - 17.3% of temporal-reasoning queries match the shape
- *   - 0% of single-session-preference, 0% of single-session-assistant
- *   - 1.4-2.6% of other types — bounded blast radius
- */
-
-const TEMPORAL_PATTERNS: ReadonlyArray<RegExp> = [
-  /\b\d+\s+(day|week|month|year)s?\s+ago\b/i,
-  /\b(last|past|next)\s+(week|month|year|monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b/i,
-  /\bwhen\s+did\b/i,
-  /\b(before|after)\s+I\b/,
-  /\bago\b/i,
-  /\b(yesterday|today|tomorrow)\b/i,
-  /\bhow\s+(long|many)\s+(days?|weeks?|months?|years?)\s+ago\b/i,
-];
-
-const COMMON_CAPS_NON_NE: ReadonlySet<string> = new Set([
-  "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday",
-  "january", "february", "march", "april", "may", "june", "july", "august",
-  "september", "october", "november", "december",
-  "i", "i'd", "i've", "i'm", "i'll",
-]);
-
-export interface QueryShape {
-  readonly hasTemporal: boolean;
-  readonly hasNamedEntity: boolean;
-}
-
-export function detectQueryShape(query: string): QueryShape {
-  if (!query) return { hasTemporal: false, hasNamedEntity: false };
-  const hasTemporal = TEMPORAL_PATTERNS.some((re) => re.test(query));
-  const hasNamedEntity = detectNamedEntity(query);
-  return { hasTemporal, hasNamedEntity };
-}
-
-function detectNamedEntity(query: string): boolean {
-  const tokens = query.split(/[\s,.;:!?()"'`]+/).filter((t) => t.length > 0);
-  if (tokens.length === 0) return false;
-  for (let i = 1; i < tokens.length; i++) {
-    const tok = tokens[i];
-    if (tok && isNamedEntityToken(tok)) return true;
-  }
-  return false;
-}
-
-function isNamedEntityToken(tok: string): boolean {
-  if (COMMON_CAPS_NON_NE.has(tok.toLowerCase())) return false;
-  if (tok.length < 2) return false;
-  if (/^[A-Z]{2,}$/.test(tok)) return true;
-  const hasUpper = /[A-Z]/.test(tok);
-  const hasLower = /[a-z]/.test(tok);
-  if (hasUpper && hasLower) return true;
-  return false;
-}