clawmem 0.7.0 → 0.7.2

package/src/merge-guards.ts

@@ -0,0 +1,266 @@
+/**
+ * Contradiction-aware merge gate (Ext 2).
+ *
+ * LLM-first contradiction check with heuristic fallback. Returns a
+ * structured `ContradictionResult` that downstream merge code uses to
+ * decide whether to merge, supersede, or link two observations.
+ *
+ * Flow:
+ *   1. `llmContradictionCheck`  — structured LLM classification; returns
+ *      null on LLM cooldown, network failure, malformed JSON, or missing
+ *      `contradictory` field.
+ *   2. `heuristicContradictionCheck` — deterministic signal on
+ *      negation asymmetry or number/date mismatch. Used as fallback when
+ *      the LLM path returns null.
+ *   3. `checkContradiction` — orchestrator. Runs LLM first, falls back
+ *      to heuristic on null. Never throws. Always returns a usable
+ *      `ContradictionResult`.
+ *
+ * Adapted from Thoth `tools/memory_tool.py:111-184` contradiction-check
+ * pattern (THOTH_EXTRACTION_PLAN.md Extraction 2).
+ *
+ * Reuses the A-MEM convention relation type `'contradicts'` (plural) —
+ * see P0 taxonomy guard at `tests/unit/contradict-taxonomy.test.ts`.
+ */
+
+import type { LLM } from "./llm.ts";
+import { extractJsonFromLLM } from "./amem.ts";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export type ContradictionSource = "llm" | "heuristic" | "unknown";
+
+export interface ContradictionResult {
+  contradictory: boolean;
+  confidence: number; // 0.0 - 1.0
+  reason?: string;
+  source: ContradictionSource;
+}
+
+/**
+ * Phase-2 contradiction handling policy. `link` (default) preserves
+ * both rows as active and sets `invalidated_by` as a backlink for
+ * operator queries. `supersede` additionally sets `invalidated_at` on
+ * the old row so it stops surfacing in active recalls.
+ */
+export type ContradictionPolicy = "link" | "supersede";
+
+export function resolveContradictionPolicy(): ContradictionPolicy {
+  const raw = process.env.CLAWMEM_CONTRADICTION_POLICY;
+  if (raw === "supersede") return "supersede";
+  return "link"; // default
+}
+
+/**
+ * Minimum LLM contradiction confidence to act on. Lower scores are
+ * treated as inconclusive and the merge proceeds (conservative: only
+ * block merges on clear contradictions). Overridable via
+ * `CLAWMEM_CONTRADICTION_MIN_CONFIDENCE` env var (0.0 - 1.0).
+ */
+export const CONTRADICTION_MIN_CONFIDENCE = parseEnvFloat(
+  "CLAWMEM_CONTRADICTION_MIN_CONFIDENCE",
+  0.5
+);
+
+function parseEnvFloat(name: string, fallback: number): number {
+  const raw = process.env[name];
+  if (raw === undefined) return fallback;
+  const n = Number.parseFloat(raw);
+  if (!Number.isFinite(n) || n < 0 || n > 1) return fallback;
+  return n;
+}
+
+// =============================================================================
+// Heuristic contradiction detection (deterministic, no LLM)
+// =============================================================================
+
+/**
+ * Deterministic heuristic contradiction check.
+ *
+ * Signals:
+ *  - **Negation asymmetry:** one side has an explicit negation token
+ *    (`not`, `never`, `no`, `didn't`, etc.) and the other doesn't.
+ *  - **Number/date mismatch:** both sides cite numbers or dates but the
+ *    sets have no shared values.
+ *
+ * Intentionally conservative: returns `contradictory=false,
+ * confidence=0` when no signal is found, leaving the decision to the
+ * LLM or the caller's default.
+ */
+export function heuristicContradictionCheck(
+  a: string,
+  b: string
+): ContradictionResult {
+  const negA = hasNegation(a);
+  const negB = hasNegation(b);
+
+  // Negation asymmetry: one side explicitly negates, the other doesn't
+  if (negA !== negB) {
+    return {
+      contradictory: true,
+      confidence: 0.6,
+      reason: "negation asymmetry — one statement has explicit negation",
+      source: "heuristic",
+    };
+  }
+
+  const numsA = extractNumbers(a);
+  const numsB = extractNumbers(b);
+
+  // Number/date mismatch: both cite numbers but no shared values
+  if (numsA.length > 0 && numsB.length > 0) {
+    const setA = new Set(numsA);
+    const setB = new Set(numsB);
+    const shared = [...setA].filter((n) => setB.has(n));
+    if (shared.length === 0) {
+      return {
+        contradictory: true,
+        confidence: 0.5,
+        reason: `number/date mismatch (A=${numsA.join(",")} B=${numsB.join(",")})`,
+        source: "heuristic",
+      };
+    }
+  }
+
+  // No heuristic signal
+  return {
+    contradictory: false,
+    confidence: 0.0,
+    reason: "no heuristic signal",
+    source: "heuristic",
+  };
+}
+
+/**
+ * Extract standalone integers, decimals, and ISO-ish dates from a
+ * string as a normalized set of numeric tokens.
+ */
+function extractNumbers(s: string): string[] {
+  // Matches: integers, decimals (1.5, 1,000), ISO dates (2026-04-10),
+  // US dates (04/10/2026), version strings (v0.7.1 → 0.7.1)
+  const matches = s.match(/\b\d{1,5}(?:[.,/-]\d{1,5}){0,2}\b/g) || [];
+  return matches.map((m) => m.replace(/,/g, ""));
+}
+
+/**
+ * Return true if the string contains an explicit negation token.
+ * Matches English contractions (didn't, won't, cannot, etc.) plus
+ * bare negations (not, never, no).
+ */
+function hasNegation(s: string): boolean {
+  return /\b(not|never|no|don['\u2019]t|didn['\u2019]t|won['\u2019]t|cannot|can['\u2019]t|wasn['\u2019]t|isn['\u2019]t|aren['\u2019]t|weren['\u2019]t|shouldn['\u2019]t|couldn['\u2019]t|wouldn['\u2019]t)\b/i.test(
+    s
+  );
+}
+
+// =============================================================================
+// LLM-based contradiction detection
+// =============================================================================
+
+const CONTRADICTION_PROMPT_TEMPLATE = `You are a logic checker. Determine whether two statements contradict each other.
+
+Statement A: {A}
+
+Statement B: {B}{CONTEXT}
+
+A contradiction exists if one statement directly denies the other, or if both cannot be true at the same time. Subtle differences in specificity (e.g. "Bob" vs "Bob Smith") are NOT contradictions. Different dates, counts, outcomes, or decisions on the same subject ARE contradictions.
+
+Respond with ONLY a JSON object:
+{"contradictory": true|false, "confidence": 0.0-1.0, "reason": "brief explanation"}
+
+Do not include any other text. /no_think`;
+
+/**
+ * LLM-based contradiction classifier.
+ *
+ * Returns `null` on any of:
+ *  - LLM generate call throws
+ *  - LLM returns null (cooldown, timeout, remote LLM down)
+ *  - LLM returns text but JSON extraction fails
+ *  - Parsed JSON is missing a boolean `contradictory` field
+ *
+ * Callers should fall back to the heuristic path on null.
+ */
+export async function llmContradictionCheck(
+  llm: LLM,
+  a: string,
+  b: string,
+  context?: string
+): Promise<ContradictionResult | null> {
+  const prompt = CONTRADICTION_PROMPT_TEMPLATE.replace("{A}", a)
+    .replace("{B}", b)
+    .replace("{CONTEXT}", context ? `\n\nContext:\n${context}` : "");
+
+  let result;
+  try {
+    result = await llm.generate(prompt, { temperature: 0.2, maxTokens: 150 });
+  } catch {
+    return null;
+  }
+
+  if (!result?.text) return null;
+
+  const parsed = extractJsonFromLLM(result.text) as {
+    contradictory?: unknown;
+    confidence?: unknown;
+    reason?: unknown;
+  } | null;
+
+  if (!parsed || typeof parsed.contradictory !== "boolean") return null;
+
+  const confidence =
+    typeof parsed.confidence === "number" && Number.isFinite(parsed.confidence)
+      ? Math.max(0, Math.min(1, parsed.confidence))
+      : 0.5;
+
+  return {
+    contradictory: parsed.contradictory,
+    confidence,
+    reason: typeof parsed.reason === "string" ? parsed.reason : undefined,
+    source: "llm",
+  };
+}
+
+// =============================================================================
+// Orchestrator
+// =============================================================================
+
+/**
+ * Orchestrated contradiction check.
+ *
+ * 1. Try LLM path; if it returns a usable result, use it.
+ * 2. Otherwise fall back to the deterministic heuristic.
+ *
+ * Never throws. Always returns a `ContradictionResult`. When the
+ * result's `source` is `heuristic` and `contradictory=false`, the
+ * caller knows the check is inconclusive and should proceed with the
+ * default merge path.
+ */
+export async function checkContradiction(
+  llm: LLM,
+  a: string,
+  b: string,
+  context?: string
+): Promise<ContradictionResult> {
+  const llmResult = await llmContradictionCheck(llm, a, b, context);
+  if (llmResult) return llmResult;
+  return heuristicContradictionCheck(a, b);
+}
+
+/**
+ * Apply the `CONTRADICTION_MIN_CONFIDENCE` threshold to a
+ * `ContradictionResult` — returns true iff the result claims a
+ * contradiction AND meets the confidence floor.
+ *
+ * Callers use this to decide whether to block a merge. Keeping the
+ * threshold check centralized means operators can tune via env var
+ * without touching the merge code.
+ */
+export function isActionableContradiction(result: ContradictionResult): boolean {
+  return (
+    result.contradictory === true &&
+    result.confidence >= CONTRADICTION_MIN_CONFIDENCE
+  );
+}

package/src/text-similarity.ts

@@ -0,0 +1,364 @@
+/**
+ * Text similarity + merge safety gate for consolidation.
+ *
+ * Prevents semantic collision between topics that share vocabulary but
+ * refer to different subjects (e.g., two observations about "Dan" vs
+ * "Dad" or "Bob" vs "Rob"). Adds a dual-threshold safety check after
+ * the cheap Jaccard candidate-generation step.
+ *
+ * Entity-aware first: uses `entity_mentions` when both sides have canonical
+ * entities resolved. Lexical fallback via proper-noun anchor regex when
+ * either side lacks entity state. Strictest default when both sides are
+ * empty (no anchors at all).
+ *
+ * Adapted from Thoth `dream_cycle.py:218-272` subject-name guard
+ * (THOTH_EXTRACTION_PLAN.md Extraction 3).
+ */
+
+import type { Store } from "./store.ts";
+
+// =============================================================================
+// Config — dual-threshold merge safety
+// =============================================================================
+
+/**
+ * NORMAL threshold: applies when anchor sets are compatible (subset or
+ * high overlap — same primary subject) AND the gate is gating on text
+ * similarity alone. Overridable via `CLAWMEM_MERGE_SCORE_NORMAL` env var
+ * for operator calibration during rollout.
+ *
+ * ⚠ Threshold is inherited from Thoth's `dream_cycle.py:218-272` guard,
+ * which uses Python's `difflib.SequenceMatcher` (character-level LCS).
+ * ClawMem uses normalized character 3-gram cosine, which is systematically
+ * harsher on benign rephrasings (word-order changes, synonym swaps). A
+ * same-meaning paraphrase like "The team migrated auth to OAuth2 last
+ * Friday" vs "Last Friday the team completed the auth migration to
+ * OAuth2" lands around 0.5 in 3-gram cosine but near 0.85 in
+ * SequenceMatcher. Consequence: merges will fragment more than Thoth
+ * did. This is the SAFE trade-off — fragmentation > false merges — but
+ * operators should tune via env var once they have real data.
+ */
+export const MERGE_SCORE_NORMAL = parseEnvFloat(
+  "CLAWMEM_MERGE_SCORE_NORMAL",
+  0.93
+);
+
+/**
+ * STRICT threshold: applies in the strictest-default path (both sides
+ * have zero anchors — no canonical entities, no proper-noun anchors).
+ * Overridable via `CLAWMEM_MERGE_SCORE_STRICT`.
+ *
+ * Non-strictest-default paths use the hard-reject rule on materially
+ * different anchors, not this threshold.
+ */
+export const MERGE_SCORE_STRICT = parseEnvFloat(
+  "CLAWMEM_MERGE_SCORE_STRICT",
+  0.98
+);
+
+function parseEnvFloat(name: string, fallback: number): number {
+  const raw = process.env[name];
+  if (raw === undefined) return fallback;
+  const n = Number.parseFloat(raw);
+  if (!Number.isFinite(n) || n < 0 || n > 1) return fallback;
+  return n;
+}
+
+// =============================================================================
+// Anchor extraction (entity-first, lexical fallback)
+// =============================================================================
+
+export type AnchorSource = "entity_mentions" | "lexical_fallback";
+
+export interface ExtractedAnchors {
+  entities: string[];
+  method: AnchorSource;
+}
+
+/**
+ * Get canonical entity IDs referenced by a set of source documents.
+ * Returns `{ entities: [], method: 'lexical_fallback' }` when no entity
+ * mentions exist for any of the given docs — the caller should then
+ * fall back to lexical anchor extraction over the raw text.
+ */
+export function extractSourceDocEntities(
+  store: Store,
+  sourceDocIds: number[]
+): ExtractedAnchors {
+  if (sourceDocIds.length === 0) {
+    return { entities: [], method: "lexical_fallback" };
+  }
+
+  const placeholders = sourceDocIds.map(() => "?").join(",");
+  let rows: { entity_id: string }[];
+  try {
+    rows = store.db
+      .prepare(
+        `SELECT DISTINCT entity_id FROM entity_mentions WHERE doc_id IN (${placeholders})`
+      )
+      .all(...sourceDocIds) as { entity_id: string }[];
+  } catch {
+    return { entities: [], method: "lexical_fallback" };
+  }
+
+  if (rows.length === 0) {
+    return { entities: [], method: "lexical_fallback" };
+  }
+
+  return {
+    entities: rows.map((r) => r.entity_id),
+    method: "entity_mentions",
+  };
+}
+
+/**
+ * Extract lexical subject anchors from raw text.
+ *
+ * Heuristic: capitalized tokens that are not common sentence-start words.
+ * This is the fallback when `entity_mentions` is empty (the doc has not
+ * been through entity enrichment yet, or is from the pre-entity era).
+ */
+export function extractSubjectAnchorsLexical(text: string): string[] {
+  if (!text) return [];
+
+  // Match capitalized tokens: CamelCase, UPPERCASE, Capitalized.
+  // Minimum 2 chars to avoid matching stray initials at sentence start.
+  const matches = text.match(/\b[A-Z][a-zA-Z0-9]{1,}\b/g) || [];
+
+  // Filter common sentence-start capitalized words that aren't proper nouns
+  const stopwords = new Set<string>([
+    "the", "a", "an", "this", "that", "these", "those",
+    "it", "we", "i", "he", "she", "they", "you", "me", "him", "her", "us", "them",
+    "and", "but", "or", "not", "is", "was", "are", "were",
+    "be", "been", "being", "have", "has", "had",
+    "do", "does", "did", "will", "would", "should", "could", "can",
+    "may", "might", "must", "shall",
+    "in", "on", "at", "to", "for", "of", "with", "by", "from",
+    "if", "then", "else", "when", "while", "where", "how", "why",
+    "all", "any", "some", "no", "one", "two",
+  ]);
+
+  const normalized = new Set<string>();
+  for (const token of matches) {
+    const lower = token.toLowerCase();
+    if (lower.length >= 2 && !stopwords.has(lower)) {
+      normalized.add(lower);
+    }
+  }
+
+  return [...normalized];
+}
+
+// =============================================================================
+// Normalized character 3-gram cosine similarity
+// =============================================================================
+
+/**
+ * Character 3-gram cosine similarity.
+ *
+ * Robust to word-level permutation and punctuation; catches near-duplicate
+ * statements that differ only in wording or whitespace. Returns 0.0..1.0.
+ *
+ * Chosen over Jaccard (used as the cheap first-stage filter) because
+ * 3-gram cosine is tighter on paraphrase detection — it distinguishes
+ * "Dan visited Paris" from "Dad visited Paris" while the Jaccard over
+ * long-word sets would treat both as near-duplicates.
+ */
+export function normalizedCosine3Gram(a: string, b: string): number {
+  const na = normalizeForTrigram(a);
+  const nb = normalizeForTrigram(b);
+
+  if (na.length === 0 || nb.length === 0) return 0;
+  if (na === nb) return 1.0;
+
+  const ta = trigramCounts(na);
+  const tb = trigramCounts(nb);
+
+  let dot = 0;
+  for (const [gram, count] of ta) {
+    const other = tb.get(gram);
+    if (other) dot += count * other;
+  }
+
+  const ma = magnitude(ta);
+  const mb = magnitude(tb);
+  if (ma === 0 || mb === 0) return 0;
+
+  return dot / (ma * mb);
+}
+
+function normalizeForTrigram(s: string): string {
+  return s
+    .toLowerCase()
+    .replace(/[^a-z0-9 ]+/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function trigramCounts(s: string): Map<string, number> {
+  const out = new Map<string, number>();
+  if (s.length < 3) {
+    out.set(s, 1);
+    return out;
+  }
+  for (let i = 0; i <= s.length - 3; i++) {
+    const gram = s.slice(i, i + 3);
+    out.set(gram, (out.get(gram) || 0) + 1);
+  }
+  return out;
+}
+
+function magnitude(m: Map<string, number>): number {
+  let sum = 0;
+  for (const v of m.values()) sum += v * v;
+  return Math.sqrt(sum);
+}
+
+// =============================================================================
+// Anchor set comparison
+// =============================================================================
+
+/**
+ * Determine whether two anchor sets "materially differ".
+ *
+ * Rules (all case-insensitive):
+ *  1. Either side empty → NOT materially different (caller handles via
+ *     strictest-default path).
+ *  2. One set is a subset of the other → NOT materially different
+ *     (allows `"Bob"` ↔ `"Bob Smith"`).
+ *  3. Intersection empty → materially different (`"Dan"` vs `"Dad"`).
+ *  4. Partial overlap → materially different when AT MOST half of the
+ *     smaller set is shared (boundary `≤ 0.5` treated as material to
+ *     fence off primary-subject mismatches like
+ *     `[alice, auth-service]` vs `[bob, auth-service]` where the only
+ *     shared anchor is the context, not the subject).
+ */
+export function anchorSetsMateriallyDiffer(a: string[], b: string[]): boolean {
+  if (a.length === 0 || b.length === 0) return false;
+
+  const setA = new Set(a.map((x) => x.toLowerCase()));
+  const setB = new Set(b.map((x) => x.toLowerCase()));
+
+  const aSubB = [...setA].every((x) => setB.has(x));
+  const bSubA = [...setB].every((x) => setA.has(x));
+  if (aSubB || bSubA) return false;
+
+  const intersect = [...setA].filter((x) => setB.has(x));
+  if (intersect.length === 0) return true;
+
+  const smaller = Math.min(setA.size, setB.size);
+  return intersect.length / smaller <= 0.5;
+}
+
+// =============================================================================
+// Merge safety gate
+// =============================================================================
+
+export type MergeSafetyMethod = "entity_aware" | "lexical_only" | "strictest_default";
+
+export interface MergeSafetyResult {
+  accepted: boolean;
+  score: number;
+  threshold: number;
+  reason: string;
+  method: MergeSafetyMethod;
+}
+
+/**
+ * Merge safety gate.
+ *
+ * Flow:
+ *  1. Compute normalized character 3-gram cosine similarity between the
+ *     candidate and existing observation texts.
+ *  2. Extract anchor sets for both sides. Entity-aware first
+ *     (`entity_mentions`), lexical fallback otherwise. If EITHER side
+ *     lacks `entity_mentions` coverage, both sides fall back to lexical
+ *     so the comparison is apples-to-apples.
+ *  3. Decide:
+ *      - Both anchor sets empty (strictest default) → accept iff
+ *        `score >= MERGE_SCORE_STRICT`.
+ *      - Anchors materially differ → **HARD REJECT regardless of text
+ *        similarity**. This is the primary safety goal: two observations
+ *        whose canonical subjects differ are never the same observation,
+ *        even if the LLM emits identical wording. Historically the gate
+ *        upgraded to a stricter threshold instead of hard-rejecting, but
+ *        that allowed merges at score 1.0 when the LLM emitted templated
+ *        text with no subject name.
+ *      - Anchors compatible (subset or high overlap) → accept iff
+ *        `score >= MERGE_SCORE_NORMAL`.
+ */
+export function passesMergeSafety(
+  store: Store,
+  candidateText: string,
+  candidateSourceDocIds: number[],
+  existingText: string,
+  existingSourceDocIds: number[]
+): MergeSafetyResult {
+  const score = normalizedCosine3Gram(candidateText, existingText);
+
+  const candEnt = extractSourceDocEntities(store, candidateSourceDocIds);
+  const existEnt = extractSourceDocEntities(store, existingSourceDocIds);
+
+  // Use entity-aware path only when BOTH sides have entity mentions —
+  // otherwise the comparison is apples-to-oranges (one side is a set of
+  // canonical IDs, the other is a set of lexical tokens).
+  const bothEntity =
+    candEnt.method === "entity_mentions" && existEnt.method === "entity_mentions";
+
+  let anchorsA: string[];
+  let anchorsB: string[];
+  let method: MergeSafetyMethod;
+
+  if (bothEntity) {
+    anchorsA = candEnt.entities;
+    anchorsB = existEnt.entities;
+    method = "entity_aware";
+  } else {
+    anchorsA = extractSubjectAnchorsLexical(candidateText);
+    anchorsB = extractSubjectAnchorsLexical(existingText);
+    method = "lexical_only";
+  }
+
+  // Strictest default: both sides empty → no subject signal at all
+  if (anchorsA.length === 0 && anchorsB.length === 0) {
+    const threshold = MERGE_SCORE_STRICT;
+    const accepted = score >= threshold;
+    return {
+      accepted,
+      score,
+      threshold,
+      reason: accepted
+        ? `strictest-default met (${score.toFixed(3)} >= ${threshold})`
+        : `strictest-default unmet (${score.toFixed(3)} < ${threshold})`,
+      method: "strictest_default",
+    };
+  }
+
+  // Hard reject on materially different anchors — this is the primary
+  // safety goal of the extraction. Applies to BOTH entity_aware and
+  // lexical_only modes so the policy is uniform.
+  if (anchorSetsMateriallyDiffer(anchorsA, anchorsB)) {
+    return {
+      accepted: false,
+      score,
+      // Reported threshold is STRICT only for operator logging; the
+      // decision was hard-reject, not threshold-gated.
+      threshold: MERGE_SCORE_STRICT,
+      reason: `${method} materially different anchors — hard reject (score=${score.toFixed(3)})`,
+      method,
+    };
+  }
+
+  // Compatible anchors (subset or high overlap): gate on text similarity
+  const threshold = MERGE_SCORE_NORMAL;
+  const accepted = score >= threshold;
+  return {
+    accepted,
+    score,
+    threshold,
+    reason: accepted
+      ? `${method} aligned anchors — ${score.toFixed(3)} >= ${threshold}`
+      : `${method} aligned anchors — ${score.toFixed(3)} < ${threshold}`,
+    method,
+  };
+}