nlm-memory 0.4.2 → 0.5.1

package/src/core/recall/recall-service.ts

@@ -1,320 +0,0 @@
-/**
- * RecallService — the use case. Composes filters, keyword scoring, and
- * semantic search into a single recall operation.
- *
- * Depends only on ports (SessionStore, LLMClient). No framework imports,
- * no SQLite, no HTTP. Tests substitute fake adapters.
- */
-
-import type { LLMClient } from "@ports/llm-client.js";
-import { LLMUnreachableError } from "@ports/llm-client.js";
-import type {
-  KeywordNeighbor,
-  SemanticNeighbor,
-  SessionStore,
-} from "@ports/session-store.js";
-import type {
-  MatchField,
-  RecallHit,
-  RecallMode,
-  RecallQuery,
-  RecallResult,
-  Session,
-} from "@shared/types.js";
-import { applyFilter } from "./filter.js";
-import { keywordMatchFields } from "./match-fields.js";
-import { detectQueryShape } from "./query-shape.js";
-import { tokenSet } from "./tokenize.js";
-
-const DEFAULT_LIMIT = 20;
-const MAX_LIMIT = 100;
-// Reciprocal Rank Fusion constant (Cormack et al. 2009). k=60 is the
-// canonical literature default. RRF combines ranked lists from multiple
-// retrievers by summing 1/(k + rank) per retriever, ignoring raw scores —
-// robust to wildly different score distributions (BM25 unbounded vs cosine
-// in [-1,1]) without requiring normalization.
-const RRF_K = 60;
-const SEMANTIC_OVERFETCH = 3;
-const KEYWORD_OVERFETCH = 3;
-
-export interface RecallServiceDeps {
-  readonly store: SessionStore;
-  readonly llm: LLMClient;
-}
-
-export class RecallService {
-  constructor(private readonly deps: RecallServiceDeps) {}
-
-  async search(input: RecallQuery): Promise<RecallResult> {
-    const mode: RecallMode = input.mode ?? "keyword";
-    const limit = clampLimit(input.limit);
-    const entity = input.entity ?? null;
-    const kind = input.kind ?? null;
-
-    const empty: RecallResult = {
-      query: input.query,
-      entity,
-      kind,
-      mode,
-      limit,
-      total: 0,
-      results: [],
-    };
-
-    if (!input.query && !entity && !kind) return empty;
-
-    // 1. Search legs — ranked neighbor IDs only. No session bodies loaded.
-    const kwNeighbors: ReadonlyArray<KeywordNeighbor> =
-      (mode === "keyword" || mode === "hybrid") && input.query
-        ? await this.deps.store.keywordSearch(input.query, limit * KEYWORD_OVERFETCH)
-        : [];
-
-    let semNeighbors: ReadonlyArray<SemanticNeighbor> = [];
-    let semError: "ollama_unreachable" | null = null;
-    if ((mode === "semantic" || mode === "hybrid") && input.query) {
-      try {
-        const embedding = await this.deps.llm.embed(input.query, "query");
-        semNeighbors = await this.deps.store.semanticSearch(
-          embedding.vector,
-          limit * SEMANTIC_OVERFETCH,
-        );
-      } catch (err) {
-        if (err instanceof LLMUnreachableError) {
-          semError = "ollama_unreachable";
-        } else {
-          throw err;
-        }
-      }
-    }
-
-    if (mode === "semantic" && semError) {
-      return { ...empty, modeUnavailable: semError };
-    }
-
-    // 2. Resolve ONLY the hit sessions — never the whole corpus. The
-    //    entity/kind filter is applied to the fetched hits; a filtered-out
-    //    session is absent from byId and is skipped during resolution.
-    const hitIds = uniqueIds(kwNeighbors, semNeighbors);
-    const hitSessions = await this.deps.store.getByIds(hitIds);
-    const filterArgs: { entity?: string; kind?: typeof input.kind } = {};
-    if (input.entity !== undefined) filterArgs.entity = input.entity;
-    if (input.kind !== undefined) filterArgs.kind = input.kind;
-    const byId = new Map<string, Session>(
-      applyFilter(hitSessions, filterArgs).map((s) => [s.id, s]),
-    );
-
-    // 3. Build hits from the resolved sessions, preserving leg rank order.
-    const queryTokens = input.query
-      ? new Set(tokenSet(input.query))
-      : new Set<string>();
-
-    const kwHits: KeywordHit[] = [];
-    for (const n of kwNeighbors) {
-      const session = byId.get(n.sessionId);
-      if (!session) continue;
-      kwHits.push({
-        session,
-        score: n.score,
-        matchedIn: keywordMatchFields(session, queryTokens),
-      });
-    }
-
-    const semHits: SemanticHit[] = [];
-    for (const n of semNeighbors) {
-      const session = byId.get(n.sessionId);
-      if (!session) continue;
-      semHits.push({ session, similarity: cosineFromL2(n.distance) });
-    }
-
-    // 4. Finalize per mode.
-    if (mode === "keyword") {
-      return finalize(input.query, entity, kind, mode, limit, kwHits.map(toKeywordHit));
-    }
-    if (mode === "semantic") {
-      return finalize(input.query, entity, kind, mode, limit, semHits.map(toSemanticHit));
-    }
-    const merged = mergeHybrid(kwHits, semHits);
-    const shape = detectQueryShape(input.query);
-    const forceIncluded = (shape.hasTemporal && shape.hasNamedEntity)
-      ? forceIncludeKeywordTop(merged, kwHits, limit)
-      : merged;
-    const result = finalize(input.query, entity, kind, mode, limit, forceIncluded);
-    return semError ? { ...result, modeUnavailable: semError } : result;
-  }
-}
-
-function uniqueIds(
-  kw: ReadonlyArray<KeywordNeighbor>,
-  sem: ReadonlyArray<SemanticNeighbor>,
-): ReadonlyArray<string> {
-  const ids = new Set<string>();
-  for (const n of kw) ids.add(n.sessionId);
-  for (const n of sem) ids.add(n.sessionId);
-  return [...ids];
-}
-
-interface KeywordHit {
-  readonly session: Session;
-  readonly score: number;
-  readonly matchedIn: ReadonlyArray<MatchField>;
-}
-
-interface SemanticHit {
-  readonly session: Session;
-  readonly similarity: number;
-}
-
-/**
- * Reciprocal Rank Fusion across the keyword + semantic legs.
- *
- * matchScore = Σ 1/(RRF_K + rank_i) for each retriever the session appears in.
- * A session at rank 1 in both retrievers therefore scores ~0.0328 (the max
- * possible with two retrievers at k=60); a session at rank 1 in one
- * retriever and absent from the other scores ~0.0164.
- *
- * keywordScore and semanticScore stay populated as min-max normalized
- * informational values so the UI can show "how strong was each leg" —
- * they're no longer used to compute matchScore.
- */
-function mergeHybrid(
-  kwHits: ReadonlyArray<KeywordHit>,
-  semHits: ReadonlyArray<SemanticHit>,
-): ReadonlyArray<RecallHit> {
-  const maxKw = Math.max(1, ...kwHits.map((h) => h.score));
-  const maxSem = Math.max(1, ...semHits.map((h) => h.similarity));
-
-  const kwRank = new Map<string, number>();
-  kwHits.forEach((h, i) => kwRank.set(h.session.id, i + 1));
-  const semRank = new Map<string, number>();
-  semHits.forEach((h, i) => semRank.set(h.session.id, i + 1));
-
-  const kwMap = new Map<string, KeywordHit>(kwHits.map((h) => [h.session.id, h]));
-  const semMap = new Map<string, SemanticHit>(semHits.map((h) => [h.session.id, h]));
-  const allIds = new Set<string>([...kwMap.keys(), ...semMap.keys()]);
-
-  const rows: RecallHit[] = [];
-  for (const id of allIds) {
-    const kw = kwMap.get(id);
-    const sem = semMap.get(id);
-    const session = (kw ?? sem)!.session;
-    const kRank = kwRank.get(id);
-    const sRank = semRank.get(id);
-    const rrf =
-      (kRank !== undefined ? 1 / (RRF_K + kRank) : 0) +
-      (sRank !== undefined ? 1 / (RRF_K + sRank) : 0);
-    const matchedIn = uniqueFields(kw?.matchedIn ?? [], sem ? (["semantic"] as MatchField[]) : []);
-    rows.push({
-      ...sessionHitFields(session),
-      matchScore: round4(rrf),
-      matchedIn,
-      keywordScore: kw ? round4(kw.score / maxKw) : 0,
-      semanticScore: sem ? round4(sem.similarity / maxSem) : 0,
-    });
-  }
-  rows.sort((a, b) => b.matchScore - a.matchScore);
-  return rows;
-}
-
-/**
- * Force-include the keyword-leg rank-1 session into the merged top-`limit`
- * result. Only invoked when the query shape (temporal + named entity)
- * indicates a Mode A pattern where pure RRF is known to demote keyword
- * winners (see query-shape.ts for diagnosis). If the rank-1 keyword session
- * is already in the limited top-N, no change. Otherwise it's inserted at
- * position `limit - 1`, displacing the lowest-confidence merged hit.
- */
-function forceIncludeKeywordTop(
-  merged: ReadonlyArray<RecallHit>,
-  kwHits: ReadonlyArray<KeywordHit>,
-  limit: number,
-): ReadonlyArray<RecallHit> {
-  if (kwHits.length === 0 || merged.length === 0) return merged;
-  const topId = kwHits[0]!.session.id;
-  const top = merged.slice(0, limit);
-  if (top.some((h) => h.id === topId)) return merged;
-  const forcedHit = merged.find((h) => h.id === topId);
-  if (!forcedHit) return merged;
-  const kept = top.slice(0, Math.max(0, limit - 1));
-  const tail = merged.slice(limit);
-  return [...kept, forcedHit, ...tail];
-}
-
-function toKeywordHit(h: KeywordHit): RecallHit {
-  return {
-    ...sessionHitFields(h.session),
-    matchScore: h.score,
-    matchedIn: h.matchedIn,
-  };
-}
-
-function toSemanticHit(h: SemanticHit): RecallHit {
-  return {
-    ...sessionHitFields(h.session),
-    matchScore: h.similarity,
-    matchedIn: ["semantic"],
-  };
-}
-
-function sessionHitFields(s: Session) {
-  return {
-    id: s.id,
-    startedAt: s.startedAt,
-    label: s.label,
-    summary: s.summary,
-    entities: s.entities,
-    decisions: s.decisions,
-    open: s.open,
-    status: s.status,
-  } as const;
-}
-
-function finalize(
-  query: string,
-  entity: string | null,
-  kind: RecallResult["kind"],
-  mode: RecallMode,
-  limit: number,
-  hits: ReadonlyArray<RecallHit>,
-): RecallResult {
-  return {
-    query,
-    entity,
-    kind,
-    mode,
-    limit,
-    total: hits.length,
-    results: hits.slice(0, limit),
-  };
-}
-
-function clampLimit(limit: number | undefined): number {
-  const n = limit ?? DEFAULT_LIMIT;
-  if (Number.isNaN(n) || n < 1) return 1;
-  return Math.min(MAX_LIMIT, Math.trunc(n));
-}
-
-function cosineFromL2(distance: number): number {
-  // session_embeddings stores unit-normalized vectors. For unit vectors,
-  // cos_sim = 1 - L2^2 / 2. Mirrors recall.py:_run_semantic.
-  const cos = 1 - (distance * distance) / 2;
-  return round4(Math.max(-1, Math.min(1, cos)));
-}
-
-function round4(value: number): number {
-  return Math.round(value * 10_000) / 10_000;
-}
-
-function uniqueFields(
-  a: ReadonlyArray<MatchField>,
-  b: ReadonlyArray<MatchField>,
-): ReadonlyArray<MatchField> {
-  const seen = new Set<MatchField>();
-  const out: MatchField[] = [];
-  for (const f of [...a, ...b]) {
-    if (!seen.has(f)) {
-      seen.add(f);
-      out.push(f);
-    }
-  }
-  return out;
-}

package/src/core/recall/recent-log.ts

@@ -1,59 +0,0 @@
-/**
- * recentLog — tail the query log for the /live observability panel.
- * Returns the last N entries in chronological order (most recent first).
- */
-
-import { readFileSync, existsSync, statSync } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
-
-export interface RecentLogEntry {
-  readonly ts: string;
-  readonly source: string;
-  readonly query: string | null;
-  readonly entity: string | null;
-  readonly kind: string | null;
-  readonly mode: string;
-  readonly limit: number;
-  readonly nResults: number;
-  readonly returnedIds: ReadonlyArray<string>;
-}
-
-function defaultLogPath(): string {
-  return process.env["NLM_QUERY_LOG"] ?? join(homedir(), ".nlm", "query_log.jsonl");
-}
-
-const TAIL_BYTES = 256 * 1024;
-
-export function recentQueryLog(limit: number, logPath: string = defaultLogPath()): RecentLogEntry[] {
-  if (!existsSync(logPath)) return [];
-  const size = statSync(logPath).size;
-  const start = Math.max(0, size - TAIL_BYTES);
-  const tail = readFileSync(logPath, { encoding: "utf8" }).slice(start);
-
-  const entries: RecentLogEntry[] = [];
-  for (const line of tail.split("\n")) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-    try {
-      const raw = JSON.parse(trimmed) as Record<string, unknown>;
-      entries.push({
-        ts: typeof raw["ts"] === "string" ? raw["ts"] : "",
-        source: typeof raw["source"] === "string" ? raw["source"] : "unknown",
-        query: typeof raw["query"] === "string" ? raw["query"] : null,
-        entity: typeof raw["entity"] === "string" ? raw["entity"] : null,
-        kind: typeof raw["kind"] === "string" ? raw["kind"] : null,
-        mode: typeof raw["mode"] === "string" ? raw["mode"] : "keyword",
-        limit: typeof raw["limit"] === "number" ? raw["limit"] : 0,
-        nResults: typeof raw["n_results"] === "number" ? raw["n_results"] : 0,
-        returnedIds: Array.isArray(raw["returned_ids"])
-          ? raw["returned_ids"].filter((x): x is string => typeof x === "string")
-          : [],
-      });
-    } catch {
-      continue;
-    }
-  }
-  entries.sort((a, b) => b.ts.localeCompare(a.ts));
-  return entries.slice(0, limit);
-}

package/src/core/recall/tokenize.ts

@@ -1,18 +0,0 @@
-/**
- * Tokenizer mirrors recall.py:_TOKEN_RE. Identical regex, lowercase normalize.
- * Pure function. The keyword scorer's parity with the Python implementation
- * starts here.
- */
-
-const TOKEN_PATTERN = /[A-Za-z0-9][A-Za-z0-9_.-]*/g;
-
-export function tokenize(text: string | null | undefined): ReadonlyArray<string> {
-  if (!text) return [];
-  const matches = text.match(TOKEN_PATTERN);
-  if (!matches) return [];
-  return matches.map((t) => t.toLowerCase());
-}
-
-export function tokenSet(text: string | null | undefined): Set<string> {
-  return new Set(tokenize(text));
-}