@remnic/core 1.1.12 → 1.1.13

package/src/dedup/semantic.ts

@@ -0,0 +1,148 @@
+/**
+ * @remnic/core — Write-time semantic dedup guard
+ *
+ * Complements the exact content-hash check in the orchestrator's write path
+ * by detecting near-duplicate candidate facts via embedding cosine similarity.
+ *
+ * The module intentionally has no dependency on the EmbeddingFallback or QMD
+ * classes directly — callers pass in a `lookup` function that returns the
+ * top-K nearest neighbors with their cosine scores. This keeps the decision
+ * logic pure and trivially testable with synthetic fixtures, and lets the
+ * orchestrator reuse whichever backend it already has wired up.
+ *
+ * Related issue: joshuaswarren/remnic#373
+ */
+
+// ── Public types ──────────────────────────────────────────────────────────────
+
+/** A single nearest-neighbor hit from the embedding backend. */
+export interface SemanticDedupHit {
+  /** Memory id of the existing neighbor. */
+  id: string;
+  /** Cosine similarity score in [0, 1]. */
+  score: number;
+  /** Optional source path, purely informational. */
+  path?: string;
+}
+
+/**
+ * Lookup function passed by the caller. Must return an array of hits sorted
+ * descending by score. Implementations should return an empty array (never
+ * throw) when the embedding backend is unavailable — the decision function
+ * treats that as "no near duplicate" (fail-open).
+ */
+export type SemanticDedupLookup = (
+  content: string,
+  limit: number,
+) => Promise<SemanticDedupHit[]>;
+
+export interface SemanticDedupOptions {
+  /** Master switch. When false, `decideSemanticDedup` always returns `keep`. */
+  enabled: boolean;
+  /** Cosine similarity threshold (0-1). ≥ threshold ⇒ treat as duplicate. */
+  threshold: number;
+  /** How many nearest neighbors to compare against. */
+  candidates: number;
+}
+
+export type SemanticDedupDecision =
+  | {
+      action: "keep";
+      reason:
+        | "disabled"
+        | "backend_unavailable"
+        | "no_candidates"
+        | "no_near_duplicate";
+      topScore?: number;
+      topId?: string;
+    }
+  | {
+      action: "skip";
+      reason: "near_duplicate";
+      topScore: number;
+      topId: string;
+      topPath?: string;
+    };
+
+// ── Pure decision function ────────────────────────────────────────────────────
+
+/**
+ * Pure decision function: given a lookup callback and options, decide whether
+ * the candidate content should be written or skipped as a near-duplicate.
+ *
+ * Contract:
+ *   - When `options.enabled` is false → always keep, reason="disabled".
+ *   - When the lookup throws (provider down / network error) → keep,
+ *     reason="backend_unavailable". Fail-open: a lookup failure must not block
+ *     writes.
+ *   - When the lookup succeeds but returns 0 hits (empty index or no
+ *     neighbors above the score floor) → keep, reason="no_candidates".
+ *     This is distinct from backend_unavailable so telemetry dashboards can
+ *     correctly distinguish "provider is down" from "index is empty".
+ *   - When the top hit's score ≥ threshold → skip with reason="near_duplicate".
+ *   - Otherwise → keep with reason="no_near_duplicate".
+ */
+export async function decideSemanticDedup(
+  content: string,
+  lookup: SemanticDedupLookup,
+  options: SemanticDedupOptions,
+): Promise<SemanticDedupDecision> {
+  if (!options.enabled) {
+    return { action: "keep", reason: "disabled" };
+  }
+  // Zero candidates means the operator has disabled the embedding lookup.
+  // Treat it identically to enabled=false so no backend call is made.
+  if (options.candidates === 0) {
+    return { action: "keep", reason: "disabled" };
+  }
+  const trimmed = typeof content === "string" ? content.trim() : "";
+  if (!trimmed) {
+    return { action: "keep", reason: "no_near_duplicate" };
+  }
+  const candidates = Math.max(1, Math.floor(options.candidates));
+  let hits: SemanticDedupHit[] = [];
+  try {
+    hits = await lookup(trimmed, candidates);
+  } catch {
+    // Fail-open: a lookup error must not block writes.
+    return { action: "keep", reason: "backend_unavailable" };
+  }
+  if (!Array.isArray(hits) || hits.length === 0) {
+    // Provider responded (no throw) but returned no hits: the embedding index
+    // is empty or contains no neighbors above the score floor. Use a distinct
+    // reason so callers and telemetry can differentiate this from a genuine
+    // backend failure.
+    return { action: "keep", reason: "no_candidates" };
+  }
+
+  // Defensive: callers ought to return sorted, but don't trust it.
+  let top: SemanticDedupHit | undefined;
+  for (const hit of hits) {
+    if (!hit || typeof hit.score !== "number" || !Number.isFinite(hit.score)) {
+      continue;
+    }
+    if (!top || hit.score > top.score) {
+      top = hit;
+    }
+  }
+  if (!top) {
+    return { action: "keep", reason: "no_near_duplicate" };
+  }
+
+  if (top.score >= options.threshold) {
+    return {
+      action: "skip",
+      reason: "near_duplicate",
+      topScore: top.score,
+      topId: top.id,
+      topPath: top.path,
+    };
+  }
+
+  return {
+    action: "keep",
+    reason: "no_near_duplicate",
+    topScore: top.score,
+    topId: top.id,
+  };
+}

package/src/delinearize.ts

@@ -0,0 +1,193 @@
+/**
+ * De-linearization transform (SimpleMem-inspired).
+ *
+ * Converts extracted memories from context-dependent to context-independent
+ * by resolving coreferences and anchoring temporal expressions.
+ */
+
+import type { EntityMention } from "./types.js";
+
+// possessive: true means the pronoun is a possessive form (his/her/its/their)
+// and should be replaced with "entity's" form.
+// group: pronouns in the same group are coreferential (he/him/his all refer to the same person)
+const PRONOUN_MAP: Record<string, { types: string[]; possessive: boolean; group: string }> = {
+  "he": { types: ["person"], possessive: false, group: "masc" },
+  "him": { types: ["person"], possessive: false, group: "masc" },
+  "his": { types: ["person"], possessive: true, group: "masc" },
+  "she": { types: ["person"], possessive: false, group: "fem" },
+  // "her" omitted — ambiguous between object ("saw her") and possessive ("her stack")
+  "they": { types: ["company", "project", "other"], possessive: false, group: "they" },
+  "them": { types: ["company", "project", "other"], possessive: false, group: "they" },
+  "their": { types: ["company", "project", "other"], possessive: true, group: "they" },
+  "it": { types: ["project", "tool", "company"], possessive: false, group: "it" },
+  "its": { types: ["project", "tool", "company"], possessive: true, group: "it" },
+};
+
+/**
+ * Replace pronouns with entity names when there's exactly one
+ * matching entity of the right type (unambiguous resolution).
+ *
+ * Possessive pronouns (his/her/its/their) become "entity's".
+ * No verb agreement is attempted — it's too fragile with adverbs/modals.
+ *
+ * Guards against:
+ * - Multiple pronouns of the same entity type → skip all (ambiguous referent)
+ * - Entity names containing pronoun substrings → placeholder-based two-pass
+ * - Entity names containing $ → escaped before replacement
+ */
+export function resolveCoReferences(fact: string, entities: EntityMention[]): string {
+  if (entities.length === 0) return fact;
+
+  // First pass: collect which pronoun entries match and which entity type they target.
+  // If multiple distinct referent groups resolve to the same entity type, skip them all
+  // (e.g., "She told him" has groups "fem" + "masc" both → person → ambiguous).
+  // Coreferential pairs like "he"+"his" share a group ("masc") and are NOT ambiguous.
+  const matchingEntries: Array<{ pronoun: string; info: { types: string[]; possessive: boolean; group: string }; entity: EntityMention }> = [];
+  const typeToGroups = new Map<string, Set<string>>();
+
+  for (const [pronoun, info] of Object.entries(PRONOUN_MAP)) {
+    // Negative lookahead for apostrophe prevents matching contractions (He's, It's, They're)
+    const regex = new RegExp(`\\b${pronoun}\\b(?![''\u2019])`, "gi");
+    if (!regex.test(fact)) continue;
+
+    const candidates = entities.filter((e) => info.types.includes(e.type));
+    if (candidates.length !== 1) continue;
+
+    matchingEntries.push({ pronoun, info, entity: candidates[0] });
+
+    // Track which referent groups target the matched entity's actual type (not all candidate types)
+    const matchedType = candidates[0].type;
+    if (!typeToGroups.has(matchedType)) typeToGroups.set(matchedType, new Set());
+    typeToGroups.get(matchedType)!.add(info.group);
+  }
+
+  // Determine which entity types have ambiguous multi-group references
+  const ambiguousTypes = new Set<string>();
+  for (const [type, groups] of typeToGroups) {
+    const matchedEntities = entities.filter((e) => e.type === type);
+    if (matchedEntities.length === 1 && groups.size > 1) {
+      ambiguousTypes.add(type);
+    }
+  }
+
+  // Filter out entries whose entity type is ambiguous
+  const safeEntries = matchingEntries.filter(
+    (e) => !e.info.types.some((t) => ambiguousTypes.has(t)),
+  );
+
+  if (safeEntries.length === 0) return fact;
+
+  // Two-pass replacement using placeholders to prevent entity names
+  // from being re-matched by subsequent pronoun patterns.
+  let result = fact;
+  const placeholders: string[] = [];
+
+  for (const { pronoun, info, entity } of safeEntries) {
+    const placeholder = `\x00ENTITY_${placeholders.length}\x00`;
+    const safeEntityName = entity.name.replace(/\$/g, "$$$$");
+    const replacement = info.possessive ? `${safeEntityName}'s` : safeEntityName;
+    placeholders.push(replacement);
+    result = result.replace(
+      new RegExp(`\\b${pronoun}\\b(?![''\u2019])`, "gi"),
+      placeholder,
+    );
+  }
+
+  // Restore placeholders with actual entity names
+  for (let i = 0; i < placeholders.length; i++) {
+    result = result.replaceAll(`\x00ENTITY_${i}\x00`, placeholders[i]);
+  }
+
+  return result;
+}
+
+function formatDate(d: Date): string {
+  return d.toISOString().slice(0, 10);
+}
+
+const TEMPORAL_PATTERNS: Array<{
+  pattern: RegExp;
+  replace: (now: Date) => string;
+// Compound patterns first (longer matches before shorter to avoid partial replacement)
+}> = [
+  {
+    pattern: /\bearlier today\b/gi,
+    replace: (now) => `earlier on ${formatDate(now)}`,
+  },
+  {
+    pattern: /\bthis morning\b/gi,
+    replace: (now) => `on the morning of ${formatDate(now)}`,
+  },
+  {
+    pattern: /\bthis afternoon\b/gi,
+    replace: (now) => `on the afternoon of ${formatDate(now)}`,
+  },
+  {
+    pattern: /\bthis evening\b/gi,
+    replace: (now) => `on the evening of ${formatDate(now)}`,
+  },
+  {
+    pattern: /\blast week\b/gi,
+    replace: (now) => {
+      const d = new Date(now);
+      d.setUTCDate(d.getUTCDate() - 7);
+      return `around ${formatDate(d)}`;
+    },
+  },
+  {
+    pattern: /\blast month\b/gi,
+    replace: (now) => {
+      const d = new Date(now);
+      d.setUTCDate(1); // avoid overflow (e.g. Mar 31 → Feb 31 → Mar 3)
+      d.setUTCMonth(d.getUTCMonth() - 1);
+      return `around ${formatDate(d)}`;
+    },
+  },
+  // Simple patterns last
+  {
+    pattern: /\byesterday\b/gi,
+    replace: (now) => {
+      const d = new Date(now);
+      d.setUTCDate(d.getUTCDate() - 1);
+      return `on ${formatDate(d)}`;
+    },
+  },
+  {
+    pattern: /\btoday\b/gi,
+    replace: (now) => `on ${formatDate(now)}`,
+  },
+  {
+    pattern: /\btomorrow\b/gi,
+    replace: (now) => {
+      const d = new Date(now);
+      d.setUTCDate(d.getUTCDate() + 1);
+      return `on ${formatDate(d)}`;
+    },
+  },
+];
+
+/**
+ * Replace relative temporal expressions with absolute ISO-8601 dates.
+ */
+export function anchorTemporalExpressions(fact: string, now: Date): string {
+  let result = fact;
+  for (const { pattern, replace } of TEMPORAL_PATTERNS) {
+    result = result.replace(pattern, replace(now));
+  }
+  return result;
+}
+
+/**
+ * Full de-linearization pipeline: temporal anchoring first, then coreference.
+ * Order matters: entity names may contain temporal words (e.g., "project-tomorrow")
+ * which would be corrupted if temporal anchoring ran after coreference insertion.
+ */
+export function delinearize(
+  factContent: string,
+  entities: EntityMention[],
+  timestamp: Date,
+): string {
+  let result = anchorTemporalExpressions(factContent, timestamp);
+  result = resolveCoReferences(result, entities);
+  return result;
+}