clawmem 0.6.0 → 0.7.1

package/src/merge-guards.ts

@@ -0,0 +1,266 @@
+/**
+ * Contradiction-aware merge gate (Ext 2).
+ *
+ * LLM-first contradiction check with heuristic fallback. Returns a
+ * structured `ContradictionResult` that downstream merge code uses to
+ * decide whether to merge, supersede, or link two observations.
+ *
+ * Flow:
+ *   1. `llmContradictionCheck`  — structured LLM classification; returns
+ *      null on LLM cooldown, network failure, malformed JSON, or missing
+ *      `contradictory` field.
+ *   2. `heuristicContradictionCheck` — deterministic signal on
+ *      negation asymmetry or number/date mismatch. Used as fallback when
+ *      the LLM path returns null.
+ *   3. `checkContradiction` — orchestrator. Runs LLM first, falls back
+ *      to heuristic on null. Never throws. Always returns a usable
+ *      `ContradictionResult`.
+ *
+ * Adapted from Thoth `tools/memory_tool.py:111-184` contradiction-check
+ * pattern (THOTH_EXTRACTION_PLAN.md Extraction 2).
+ *
+ * Reuses the A-MEM convention relation type `'contradicts'` (plural) —
+ * see P0 taxonomy guard at `tests/unit/contradict-taxonomy.test.ts`.
+ */
+
+import type { LLM } from "./llm.ts";
+import { extractJsonFromLLM } from "./amem.ts";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export type ContradictionSource = "llm" | "heuristic" | "unknown";
+
+export interface ContradictionResult {
+  contradictory: boolean;
+  confidence: number; // 0.0 - 1.0
+  reason?: string;
+  source: ContradictionSource;
+}
+
+/**
+ * Phase-2 contradiction handling policy. `link` (default) preserves
+ * both rows as active and sets `invalidated_by` as a backlink for
+ * operator queries. `supersede` additionally sets `invalidated_at` on
+ * the old row so it stops surfacing in active recalls.
+ */
+export type ContradictionPolicy = "link" | "supersede";
+
+export function resolveContradictionPolicy(): ContradictionPolicy {
+  const raw = process.env.CLAWMEM_CONTRADICTION_POLICY;
+  if (raw === "supersede") return "supersede";
+  return "link"; // default
+}
+
+/**
+ * Minimum LLM contradiction confidence to act on. Lower scores are
+ * treated as inconclusive and the merge proceeds (conservative: only
+ * block merges on clear contradictions). Overridable via
+ * `CLAWMEM_CONTRADICTION_MIN_CONFIDENCE` env var (0.0 - 1.0).
+ */
+export const CONTRADICTION_MIN_CONFIDENCE = parseEnvFloat(
+  "CLAWMEM_CONTRADICTION_MIN_CONFIDENCE",
+  0.5
+);
+
+function parseEnvFloat(name: string, fallback: number): number {
+  const raw = process.env[name];
+  if (raw === undefined) return fallback;
+  const n = Number.parseFloat(raw);
+  if (!Number.isFinite(n) || n < 0 || n > 1) return fallback;
+  return n;
+}
+
+// =============================================================================
+// Heuristic contradiction detection (deterministic, no LLM)
+// =============================================================================
+
+/**
+ * Deterministic heuristic contradiction check.
+ *
+ * Signals:
+ *  - **Negation asymmetry:** one side has an explicit negation token
+ *    (`not`, `never`, `no`, `didn't`, etc.) and the other doesn't.
+ *  - **Number/date mismatch:** both sides cite numbers or dates but the
+ *    sets have no shared values.
+ *
+ * Intentionally conservative: returns `contradictory=false,
+ * confidence=0` when no signal is found, leaving the decision to the
+ * LLM or the caller's default.
+ */
+export function heuristicContradictionCheck(
+  a: string,
+  b: string
+): ContradictionResult {
+  const negA = hasNegation(a);
+  const negB = hasNegation(b);
+
+  // Negation asymmetry: one side explicitly negates, the other doesn't
+  if (negA !== negB) {
+    return {
+      contradictory: true,
+      confidence: 0.6,
+      reason: "negation asymmetry — one statement has explicit negation",
+      source: "heuristic",
+    };
+  }
+
+  const numsA = extractNumbers(a);
+  const numsB = extractNumbers(b);
+
+  // Number/date mismatch: both cite numbers but no shared values
+  if (numsA.length > 0 && numsB.length > 0) {
+    const setA = new Set(numsA);
+    const setB = new Set(numsB);
+    const shared = [...setA].filter((n) => setB.has(n));
+    if (shared.length === 0) {
+      return {
+        contradictory: true,
+        confidence: 0.5,
+        reason: `number/date mismatch (A=${numsA.join(",")} B=${numsB.join(",")})`,
+        source: "heuristic",
+      };
+    }
+  }
+
+  // No heuristic signal
+  return {
+    contradictory: false,
+    confidence: 0.0,
+    reason: "no heuristic signal",
+    source: "heuristic",
+  };
+}
+
+/**
+ * Extract standalone integers, decimals, and ISO-ish dates from a
+ * string as a normalized set of numeric tokens.
+ */
+function extractNumbers(s: string): string[] {
+  // Matches: integers, decimals (1.5, 1,000), ISO dates (2026-04-10),
+  // US dates (04/10/2026), version strings (v0.7.1 → 0.7.1)
+  const matches = s.match(/\b\d{1,5}(?:[.,/-]\d{1,5}){0,2}\b/g) || [];
+  return matches.map((m) => m.replace(/,/g, ""));
+}
+
+/**
+ * Return true if the string contains an explicit negation token.
+ * Matches English contractions (didn't, won't, cannot, etc.) plus
+ * bare negations (not, never, no).
+ */
+function hasNegation(s: string): boolean {
+  return /\b(not|never|no|don['\u2019]t|didn['\u2019]t|won['\u2019]t|cannot|can['\u2019]t|wasn['\u2019]t|isn['\u2019]t|aren['\u2019]t|weren['\u2019]t|shouldn['\u2019]t|couldn['\u2019]t|wouldn['\u2019]t)\b/i.test(
+    s
+  );
+}
+
+// =============================================================================
+// LLM-based contradiction detection
+// =============================================================================
+
+const CONTRADICTION_PROMPT_TEMPLATE = `You are a logic checker. Determine whether two statements contradict each other.
+
+Statement A: {A}
+
+Statement B: {B}{CONTEXT}
+
+A contradiction exists if one statement directly denies the other, or if both cannot be true at the same time. Subtle differences in specificity (e.g. "Bob" vs "Bob Smith") are NOT contradictions. Different dates, counts, outcomes, or decisions on the same subject ARE contradictions.
+
+Respond with ONLY a JSON object:
+{"contradictory": true|false, "confidence": 0.0-1.0, "reason": "brief explanation"}
+
+Do not include any other text. /no_think`;
+
+/**
+ * LLM-based contradiction classifier.
+ *
+ * Returns `null` on any of:
+ *  - LLM generate call throws
+ *  - LLM returns null (cooldown, timeout, remote LLM down)
+ *  - LLM returns text but JSON extraction fails
+ *  - Parsed JSON is missing a boolean `contradictory` field
+ *
+ * Callers should fall back to the heuristic path on null.
+ */
+export async function llmContradictionCheck(
+  llm: LLM,
+  a: string,
+  b: string,
+  context?: string
+): Promise<ContradictionResult | null> {
+  const prompt = CONTRADICTION_PROMPT_TEMPLATE.replace("{A}", a)
+    .replace("{B}", b)
+    .replace("{CONTEXT}", context ? `\n\nContext:\n${context}` : "");
+
+  let result;
+  try {
+    result = await llm.generate(prompt, { temperature: 0.2, maxTokens: 150 });
+  } catch {
+    return null;
+  }
+
+  if (!result?.text) return null;
+
+  const parsed = extractJsonFromLLM(result.text) as {
+    contradictory?: unknown;
+    confidence?: unknown;
+    reason?: unknown;
+  } | null;
+
+  if (!parsed || typeof parsed.contradictory !== "boolean") return null;
+
+  const confidence =
+    typeof parsed.confidence === "number" && Number.isFinite(parsed.confidence)
+      ? Math.max(0, Math.min(1, parsed.confidence))
+      : 0.5;
+
+  return {
+    contradictory: parsed.contradictory,
+    confidence,
+    reason: typeof parsed.reason === "string" ? parsed.reason : undefined,
+    source: "llm",
+  };
+}
+
+// =============================================================================
+// Orchestrator
+// =============================================================================
+
+/**
+ * Orchestrated contradiction check.
+ *
+ * 1. Try LLM path; if it returns a usable result, use it.
+ * 2. Otherwise fall back to the deterministic heuristic.
+ *
+ * Never throws. Always returns a `ContradictionResult`. When the
+ * result's `source` is `heuristic` and `contradictory=false`, the
+ * caller knows the check is inconclusive and should proceed with the
+ * default merge path.
+ */
+export async function checkContradiction(
+  llm: LLM,
+  a: string,
+  b: string,
+  context?: string
+): Promise<ContradictionResult> {
+  const llmResult = await llmContradictionCheck(llm, a, b, context);
+  if (llmResult) return llmResult;
+  return heuristicContradictionCheck(a, b);
+}
+
+/**
+ * Apply the `CONTRADICTION_MIN_CONFIDENCE` threshold to a
+ * `ContradictionResult` — returns true iff the result claims a
+ * contradiction AND meets the confidence floor.
+ *
+ * Callers use this to decide whether to block a merge. Keeping the
+ * threshold check centralized means operators can tune via env var
+ * without touching the merge code.
+ */
+export function isActionableContradiction(result: ContradictionResult): boolean {
+  return (
+    result.contradictory === true &&
+    result.confidence >= CONTRADICTION_MIN_CONFIDENCE
+  );
+}

package/src/recall-attribution.ts

@@ -0,0 +1,182 @@
+/**
+ * Recall Attribution — per-turn reference detection for recall tracking.
+ *
+ * Extracted into a standalone module for testability (per GPT 5.4 High review turn 4).
+ *
+ * Architecture:
+ * 1. Segment the transcript into ordered turns (user → assistant pairs)
+ * 2. Zip context_usage rows (by turn_index) with transcript turns (by position)
+ * 3. For each pair, detect references in that turn's assistant text only
+ * 4. Mark recall_events linked to the usage rows whose turn actually cited the doc
+ */
+
+import type { Store, UsageRow } from "./store.ts";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export type TranscriptTurn = {
+  userText: string;
+  assistantText: string;
+};
+
+// =============================================================================
+// Transcript Segmentation
+// =============================================================================
+
+/**
+ * Segment a flat message array into ordered turns.
+ * A turn starts on each "user" message and includes all following "assistant"
+ * messages until the next "user" message.
+ *
+ * @param messages - Ordered array of {role, content} from transcript JSONL
+ * @returns Ordered array of turns
+ */
+export function segmentTranscriptIntoTurns(
+  messages: { role: string; content: string }[]
+): TranscriptTurn[] {
+  const turns: TranscriptTurn[] = [];
+  let currentUser = "";
+  let currentAssistant = "";
+
+  for (const msg of messages) {
+    if (msg.role === "user") {
+      // New turn: flush previous if it has assistant content
+      if (currentUser || currentAssistant) {
+        turns.push({ userText: currentUser, assistantText: currentAssistant });
+      }
+      currentUser = msg.content;
+      currentAssistant = "";
+    } else if (msg.role === "assistant") {
+      currentAssistant += (currentAssistant ? "\n" : "") + msg.content;
+    }
+    // Ignore system/tool messages for attribution purposes
+  }
+
+  // Flush final turn
+  if (currentUser || currentAssistant) {
+    turns.push({ userText: currentUser, assistantText: currentAssistant });
+  }
+
+  return turns;
+}
+
+// =============================================================================
+// Per-Turn Reference Detection
+// =============================================================================
+
+/**
+ * Check if a displayPath (collection/path) is referenced in text.
+ * Matches by: full path, filename (without extension), or doc title.
+ */
+function isPathReferenced(
+  store: Store,
+  displayPath: string,
+  text: string
+): boolean {
+  if (!text || !displayPath) return false;
+
+  // Full path match
+  if (text.includes(displayPath)) return true;
+
+  // Filename match (without extension, min 4 chars)
+  const filename = displayPath.split("/").pop()?.replace(/\.(md|txt)$/i, "");
+  if (filename && filename.length > 3 && text.toLowerCase().includes(filename.toLowerCase())) {
+    return true;
+  }
+
+  // Title match from DB
+  const parts = displayPath.split("/");
+  if (parts.length >= 2) {
+    const collection = parts[0]!;
+    const docPath = parts.slice(1).join("/");
+    const doc = store.findActiveDocument(collection, docPath);
+    if (doc?.title && doc.title.length >= 5 && text.toLowerCase().includes(doc.title.toLowerCase())) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+// =============================================================================
+// Attribution Core
+// =============================================================================
+
+/**
+ * Attribute recall events to specific turns using per-turn reference detection.
+ *
+ * For each context_usage row (ordered by turn_index), finds the corresponding
+ * transcript turn and checks which of that turn's injected docs were cited in
+ * that turn's assistant text. Only marks recall_events linked to turns where
+ * the doc was actually referenced.
+ *
+ * @param store - Store instance for doc resolution and event marking
+ * @param sessionId - Session identifier
+ * @param usages - context_usage rows for this session, ordered by turn_index
+ * @param turns - Transcript turns, ordered by position
+ */
+export function attributeRecallReferences(
+  store: Store,
+  sessionId: string,
+  usages: UsageRow[],
+  turns: TranscriptTurn[]
+): void {
+  // Filter to context-surfacing usages only
+  const surfacingUsages = usages.filter(u => u.hookName === "context-surfacing");
+
+  for (const usage of surfacingUsages) {
+    // Match usage to transcript turn by turn_index
+    const turn = turns[usage.turnIndex];
+    if (!turn || !turn.assistantText) continue;
+
+    // Parse injected paths for this turn
+    let injectedPaths: string[];
+    try { injectedPaths = JSON.parse(usage.injectedPaths) as string[]; }
+    catch { continue; }
+    if (injectedPaths.length === 0) continue;
+
+    // Check which docs from THIS turn were referenced in THIS turn's assistant text
+    const referencedDocIds: number[] = [];
+    for (const path of injectedPaths) {
+      if (!isPathReferenced(store, path, turn.assistantText)) continue;
+
+      const parts = path.split("/");
+      if (parts.length < 2) continue;
+      const collection = parts[0]!;
+      const docPath = parts.slice(1).join("/");
+      const doc = store.findActiveDocument(collection, docPath);
+      if (doc) referencedDocIds.push(doc.id);
+    }
+
+    if (referencedDocIds.length === 0) continue;
+
+    // Mark only recall events linked to THIS usage row
+    for (const docId of referencedDocIds) {
+      // Primary: usage_id-linked events (current schema)
+      const linked = store.db.prepare(`
+        SELECT id FROM recall_events
+        WHERE usage_id = ? AND doc_id = ? AND was_referenced = 0
+      `).all(usage.id, docId) as { id: number }[];
+
+      if (linked.length > 0) {
+        const ids = linked.map(r => r.id);
+        const placeholders = ids.map(() => "?").join(",");
+        store.db.prepare(`
+          UPDATE recall_events SET was_referenced = 1
+          WHERE id IN (${placeholders})
+        `).run(...ids);
+      } else {
+        // Fallback: pre-migration events without usage_id — match by turn_index
+        store.db.prepare(`
+          UPDATE recall_events SET was_referenced = 1
+          WHERE id IN (
+            SELECT id FROM recall_events
+            WHERE session_id = ? AND doc_id = ? AND turn_index = ? AND was_referenced = 0
+          )
+        `).run(sessionId, docId, usage.turnIndex);
+      }
+    }
+  }
+}

package/src/recall-buffer.ts

@@ -0,0 +1,85 @@
+/**
+ * Recall Tracking — direct-write recall event recording.
+ *
+ * Context-surfacing writes recall events directly to SQLite (single transaction,
+ * <0.4ms for ~12 rows). This replaces the original in-memory buffer design which
+ * failed in Claude Code mode where each hook is a separate process invocation.
+ *
+ * Per GPT 5.4 High review (Codex turn 1):
+ * - Direct INSERT is preferred over buffer for cross-process correctness
+ * - WAL mode handles concurrent writes safely (busy_timeout=5000ms)
+ * - Negative signals (surfaced but not referenced) marked retroactively by feedback-loop
+ */
+
+import { createHash } from "crypto";
+import type { Store } from "./store.ts";
+
+// =============================================================================
+// Query Hashing
+// =============================================================================
+
+/**
+ * Hash a query string for recall tracking.
+ * SHA1 truncated to 12 hex chars (same as OpenClaw's approach).
+ */
+export function hashQuery(query: string): string {
+  return createHash("sha1")
+    .update(query.toLowerCase().trim())
+    .digest("hex")
+    .slice(0, 12);
+}
+
+// =============================================================================
+// Direct Write (replaces in-memory buffer)
+// =============================================================================
+
+/**
+ * Record surfaced documents as recall events directly to SQLite.
+ * Called from context-surfacing hook — single transaction, ~0.4ms.
+ *
+ * Resolves displayPath → doc_id inline. Docs that can't be resolved
+ * (deleted between search and write) are silently skipped.
+ *
+ * @param store - Store instance with DB access
+ * @param sessionId - Current session identifier
+ * @param queryHash - SHA1 hash of the search query
+ * @param docs - Array of {displayPath, searchScore} for each surfaced result
+ * @returns Number of events recorded
+ */
+export function writeRecallEvents(
+  store: Store,
+  sessionId: string,
+  queryHash: string,
+  docs: { displayPath: string; searchScore: number }[],
+  usageId?: number,
+  turnIndex?: number
+): number {
+  if (!sessionId || docs.length === 0) return 0;
+
+  const resolved: { docId: number; queryHash: string; searchScore: number; sessionId: string }[] = [];
+
+  for (const doc of docs) {
+    const parts = doc.displayPath.split("/");
+    if (parts.length < 2) continue;
+    const collection = parts[0]!;
+    const docPath = parts.slice(1).join("/");
+    const found = store.findActiveDocument(collection, docPath);
+    if (!found) {
+      console.debug?.(`[recall] skipping unresolvable displayPath: ${doc.displayPath}`);
+      continue;
+    }
+
+    resolved.push({
+      docId: found.id,
+      queryHash,
+      searchScore: doc.searchScore,
+      sessionId,
+      usageId,
+      turnIndex,
+    });
+  }
+
+  if (resolved.length === 0) return 0;
+  return store.insertRecallEvents(resolved);
+}
+