clawmem 0.6.0 → 0.7.1

package/src/hooks/context-surfacing.ts

@@ -30,6 +30,7 @@ import { enrichResults } from "../search-utils.ts";
 import { sanitizeSnippet } from "../promptguard.ts";
 import { shouldSkipRetrieval, isRetrievedNoise } from "../retrieval-gate.ts";
 import { MAX_QUERY_LENGTH } from "../limits.ts";
+import { writeRecallEvents, hashQuery } from "../recall-buffer.ts";
 
 // =============================================================================
 // Config
@@ -57,6 +58,17 @@ const NUDGE_INTERVAL = parseInt(process.env.CLAWMEM_NUDGE_INTERVAL || "15", 10);
 const LIFECYCLE_HOOK_NAMES = ["memory_pin", "memory_forget", "memory_snooze", "lifecycle-archive"];
 const NUDGE_TEXT = "You haven't managed memory recently. If vault-context is surfacing noise → snooze it. If a critical decision was just made → pin it. If stale knowledge appeared → forget it.";
 
+// Ext 6a: Context instruction + relationship snippets
+// The instruction is ALWAYS prepended when the hook emits context — it frames
+// the surfaced facts as background knowledge the agent already holds, reducing
+// prompt-level ambiguity. Relationship snippets are fetched from the vault
+// knowledge graph for edges where BOTH endpoints are in the surfaced doc set.
+const INSTRUCTION_TEXT = "Treat the following as background facts you already know unless the user corrects them.";
+const INSTRUCTION_XML = `<instruction>${INSTRUCTION_TEXT}</instruction>`;
+const INSTRUCTION_TOKEN_COST = estimateTokens(INSTRUCTION_XML);
+const RELATIONSHIPS_XML_OVERHEAD_TOKENS = estimateTokens("<relationships>\n\n</relationships>");
+const MAX_RELATION_SNIPPETS = 10;
+
 // File path patterns to extract from prompts (E13 replacement: file-aware UserPromptSubmit)
 const FILE_PATH_RE = /(?:^|\s)((?:\/[\w.@-]+)+(?:\.\w+)?|[\w.@-]+\.(?:ts|js|py|md|sh|yaml|yml|json|toml|rs|go|tsx|jsx|css|html))\b/g;
 
@@ -69,18 +81,44 @@ export async function contextSurfacing(
   input: HookInput
 ): Promise<HookOutput> {
   let prompt = input.prompt?.trim();
-  if (!prompt || prompt.length < MIN_PROMPT_LENGTH) return makeEmptyOutput("context-surfacing");
+
+  // Compute turn_index FIRST, before any early returns.
+  // Every transcript-visible early return must log an empty context_usage row
+  // to keep turn_index aligned with transcript turns for per-turn attribution.
+  if (input.sessionId) {
+    try {
+      let turnIndex = 0;
+      try {
+        const existing = store.db.prepare(
+          `SELECT COUNT(*) as cnt FROM context_usage WHERE session_id = ? AND hook_name = 'context-surfacing'`
+        ).get(input.sessionId) as { cnt: number };
+        turnIndex = existing.cnt;
+      } catch { /* fallback to 0 */ }
+      (input as any)._turnIndex = turnIndex;
+    } catch { /* non-fatal */ }
+  }
+
+  if (!prompt || prompt.length < MIN_PROMPT_LENGTH) {
+    logEmptyTurn(store, input);
+    return makeEmptyOutput("context-surfacing");
+  }
 
   // Bound query length to prevent DoS on search indices
   if (prompt.length > MAX_QUERY_LENGTH) prompt = prompt.slice(0, MAX_QUERY_LENGTH);
 
-  // Skip slash commands
-  if (prompt.startsWith("/")) return makeEmptyOutput("context-surfacing");
+  // Skip slash commands — log empty turn for alignment
+  if (prompt.startsWith("/")) {
+    logEmptyTurn(store, input);
+    return makeEmptyOutput("context-surfacing");
+  }
 
   // Adaptive retrieval gate: skip greetings, shell commands, affirmations, etc.
-  if (shouldSkipRetrieval(prompt)) return makeEmptyOutput("context-surfacing");
+  if (shouldSkipRetrieval(prompt)) {
+    logEmptyTurn(store, input);
+    return makeEmptyOutput("context-surfacing");
+  }
 
-  // Heartbeat / duplicate suppression (IO4)
+  // Heartbeat / duplicate suppression (IO4) — NOT transcript-visible user turns
   if (isHeartbeatPrompt(prompt)) return makeEmptyOutput("context-surfacing");
   if (wasPromptSeenRecently(store, "context-surfacing", prompt)) {
     return makeEmptyOutput("context-surfacing");
@@ -157,7 +195,7 @@ export async function contextSurfacing(
     }
   }
 
-  if (results.length === 0) return makeEmptyOutput("context-surfacing");
+  if (results.length === 0) { logEmptyTurn(store, input); return makeEmptyOutput("context-surfacing"); }
 
   // Budget-aware deep escalation (deep profile only):
   // If the fast path finished quickly and found results, spend remaining time budget
@@ -215,7 +253,7 @@ export async function contextSurfacing(
     !FILTERED_PATHS.some(p => r.displayPath.includes(p))
   );
 
-  if (results.length === 0) return makeEmptyOutput("context-surfacing");
+  if (results.length === 0) { logEmptyTurn(store, input); return makeEmptyOutput("context-surfacing"); }
 
   // Filter out snoozed documents
   const now = new Date();
@@ -231,7 +269,7 @@ export async function contextSurfacing(
     return true;
   });
 
-  if (results.length === 0) return makeEmptyOutput("context-surfacing");
+  if (results.length === 0) { logEmptyTurn(store, input); return makeEmptyOutput("context-surfacing"); }
 
   // Deduplicate by filepath (keep best score per path)
   const deduped = new Map<string, SearchResult>();
@@ -273,7 +311,7 @@ export async function contextSurfacing(
       : 0;
 
     // Activation floor: if even the best result is too weak, bail entirely
-    if (bestScore < profile.activationFloor) return makeEmptyOutput("context-surfacing");
+    if (bestScore < profile.activationFloor) { logEmptyTurn(store, input); return makeEmptyOutput("context-surfacing"); }
 
     const adaptiveMin = Math.max(bestScore * profile.minScoreRatio, profile.absoluteFloor);
     scored = allScored.filter(r => r.compositeScore >= adaptiveMin);
@@ -282,7 +320,7 @@ export async function contextSurfacing(
     scored = allScored.filter(r => r.compositeScore >= minScore);
   }
 
-  if (scored.length === 0) return makeEmptyOutput("context-surfacing");
+  if (scored.length === 0) { logEmptyTurn(store, input); return makeEmptyOutput("context-surfacing"); }
 
   // Spreading activation (E11): boost results co-activated with top HOT results
   if (scored.length > 3) {
@@ -322,14 +360,70 @@ export async function contextSurfacing(
     }
   }
 
-  // Build context within token budget (profile-driven)
-  const { context, paths, tokens } = buildContext(scored, prompt, tokenBudget);
+  // Build context within token budget (profile-driven).
+  // Ext 6a: Reserve budget for the always-on instruction line so the final
+  // vault-context payload stays within `tokenBudget`. Relations are layered
+  // in afterward using whatever budget remains and are the first thing
+  // truncated when the payload would overflow.
+  const factsBudget = Math.max(0, tokenBudget - INSTRUCTION_TOKEN_COST);
+  const { context, paths, tokens } = buildContext(scored, prompt, factsBudget);
 
-  if (!context) return makeEmptyOutput("context-surfacing");
+  if (!context) {
+    logEmptyTurn(store, input);
+    return makeEmptyOutput("context-surfacing");
+  }
 
-  // Log the injection
+  // Use pre-computed turn_index from top of function
   if (input.sessionId) {
-    logInjection(store, input.sessionId, "context-surfacing", paths, tokens);
+    const turnIndex = (input as any)._turnIndex ?? 0;
+
+    // Log the injection — returns usage_id for recall event linkage
+    const usageId = logInjection(store, input.sessionId, "context-surfacing", paths, tokens, turnIndex);
+
+    // Record recall events ONLY for docs that made it into the injected context
+    // (post-budget). Docs trimmed by token budget were never seen by the model.
+    // Each event links to its context_usage row via usage_id + turn_index.
+    // Multi-vault: route docs to origin vault's store. Mirror context_usage there too.
+    try {
+      const qHash = hashQuery(prompt);
+      const injectedSet = new Set(paths);
+      const injectedScored = scored.filter(r => injectedSet.has(r.displayPath));
+
+      // Group by vault origin (undefined = general vault)
+      const byVault = new Map<string | undefined, typeof injectedScored>();
+      for (const r of injectedScored) {
+        const vault = (r as any)._fromVault as string | undefined;
+        let group = byVault.get(vault);
+        if (!group) { group = []; byVault.set(vault, group); }
+        group.push(r);
+      }
+
+      const validUsageId = usageId > 0 ? usageId : undefined;
+      for (const [vault, docs] of byVault) {
+        const mappedDocs = docs.map(r => ({ displayPath: r.displayPath, searchScore: r.compositeScore }));
+        if (!vault) {
+          writeRecallEvents(store, input.sessionId, qHash, mappedDocs, validUsageId, turnIndex);
+        } else {
+          try {
+            const vaultStore = resolveStore(vault);
+            // Mirror context_usage row into named vault for correct FK + attribution
+            const vaultPaths = docs.map(r => r.displayPath);
+            const vaultUsageId = vaultStore.insertUsage({
+              sessionId: input.sessionId,
+              timestamp: new Date().toISOString(),
+              hookName: "context-surfacing",
+              injectedPaths: vaultPaths,
+              estimatedTokens: 0,
+              wasReferenced: 0,
+              turnIndex,
+            });
+            writeRecallEvents(vaultStore, input.sessionId, qHash, mappedDocs, vaultUsageId > 0 ? vaultUsageId : undefined, turnIndex);
+          } catch { /* vault unavailable — skip */ }
+        }
+      }
+    } catch {
+      // Non-critical — don't block context surfacing on recall tracking errors
+    }
   }
 
   // Routing hint: detect query intent signals and prepend a tool routing directive
@@ -339,9 +433,29 @@ export async function contextSurfacing(
   // Memory nudge: periodically remind agent to use lifecycle tools
   const nudge = NUDGE_INTERVAL > 0 ? shouldNudge(store) : null;
 
+  // Ext 6a: Enrich vault-context with instruction framing + optional
+  // relationship snippets sourced from memory_relations. Only edges where
+  // BOTH endpoints are in the surfaced doc set are included. The relations
+  // block is the first thing dropped when the payload would overflow budget.
+  //
+  // Budget accounting (Turn 11 fix): `tokens` from buildContext only sums per-
+  // entry bodies and misses both the `<facts>...</facts>` wrapper and the
+  // `\n\n---\n\n` separators between entries. Compute the wrapped-facts cost
+  // directly from the rendered string so the relationships block can never
+  // push the final `<vault-context>` inner payload past `tokenBudget`.
+  const surfacedDocIds = lookupSurfacedDocIds(store, paths);
+  const relationSnippets = fetchRelationSnippets(store, surfacedDocIds);
+  const factsBlockXml = `<facts>\n${context}\n</facts>`;
+  const factsWrappedTokens = estimateTokens(factsBlockXml);
+  const relationBudget = Math.max(
+    0,
+    tokenBudget - INSTRUCTION_TOKEN_COST - factsWrappedTokens
+  );
+  const vaultInner = buildVaultContextInner(context, relationSnippets, relationBudget);
+
   const parts: string[] = [];
   if (routingHint) parts.push(`<vault-routing>${routingHint}</vault-routing>`);
-  parts.push(`<vault-context>\n${context}\n</vault-context>`);
+  parts.push(`<vault-context>\n${vaultInner}\n</vault-context>`);
   if (nudge) parts.push(`<vault-nudge>${NUDGE_TEXT}</vault-nudge>`);
 
   return makeContextOutput("context-surfacing", parts.join("\n"));
@@ -351,6 +465,19 @@ export async function contextSurfacing(
 // Helpers
 // =============================================================================
 
+/**
+ * Log an empty context_usage row for a skipped turn.
+ * Keeps turn_index aligned with transcript turns so per-turn recall
+ * attribution doesn't drift when some prompts are gated.
+ */
+function logEmptyTurn(store: Store, input: HookInput): void {
+  if (!input.sessionId) return;
+  try {
+    const turnIndex = (input as any)._turnIndex ?? 0;
+    logInjection(store, input.sessionId, "context-surfacing", [], 0, turnIndex);
+  } catch { /* non-fatal */ }
+}
+
 /**
  * Detect causal/temporal/discovery signals in the prompt and return a
  * routing hint that makes the correct tool choice salient at the moment
@@ -431,6 +558,148 @@ function buildContext(
   };
 }
 
+// =============================================================================
+// Ext 6a: Relationship snippets + instruction framing
+// =============================================================================
+
+/**
+ * Relationship snippet derived from a memory_relations edge whose source and
+ * target are both active documents currently surfaced by the context hook.
+ */
+export interface RelationSnippet {
+  sourceTitle: string;
+  targetTitle: string;
+  relationType: string;
+}
+
+/**
+ * Resolve surfaced display paths back to document ids so the relation query
+ * can filter memory_relations edges to the surfaced set. Silently drops paths
+ * that don't match an active row in the general vault (e.g. skill-vault paths
+ * or deactivated docs) — fail-open, never throws.
+ */
+export function lookupSurfacedDocIds(
+  store: Store,
+  displayPaths: string[]
+): number[] {
+  if (displayPaths.length === 0) return [];
+  try {
+    const placeholders = displayPaths.map(() => "?").join(",");
+    const rows = store.db
+      .prepare(
+        `SELECT id FROM documents
+         WHERE active = 1
+           AND (collection || '/' || path) IN (${placeholders})`
+      )
+      .all(...displayPaths) as Array<{ id: number }>;
+    return rows.map((r) => r.id);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Fetch relationship snippets for edges where BOTH endpoints are in the
+ * surfaced doc set. Returns an empty list on empty input, zero/one surfaced
+ * docs, self-loops, or any DB error (fail-open, never throws). Results are
+ * ordered by relation weight DESC then recency so the most salient edges
+ * survive budget truncation.
+ */
+export function fetchRelationSnippets(
+  store: Store,
+  surfacedDocIds: number[],
+  limit: number = MAX_RELATION_SNIPPETS
+): RelationSnippet[] {
+  if (surfacedDocIds.length < 2) return [];
+  try {
+    const placeholders = surfacedDocIds.map(() => "?").join(",");
+    const rows = store.db
+      .prepare(
+        `SELECT mr.relation_type,
+                ds.title AS source_title,
+                dt.title AS target_title
+         FROM memory_relations mr
+         JOIN documents ds ON ds.id = mr.source_id AND ds.active = 1
+         JOIN documents dt ON dt.id = mr.target_id AND dt.active = 1
+         WHERE mr.source_id IN (${placeholders})
+           AND mr.target_id IN (${placeholders})
+           AND mr.source_id != mr.target_id
+         ORDER BY mr.weight DESC, mr.created_at DESC
+         LIMIT ?`
+      )
+      .all(...surfacedDocIds, ...surfacedDocIds, limit) as Array<{
+      relation_type: string;
+      source_title: string;
+      target_title: string;
+    }>;
+    return rows.map((r) => ({
+      sourceTitle: r.source_title,
+      targetTitle: r.target_title,
+      relationType: r.relation_type,
+    }));
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Render relationship snippets as bullet lines, sanitizing titles to block
+ * prompt-injection via metadata fields. Lines that become filtered-content
+ * markers after sanitization are dropped.
+ */
+export function renderRelationshipLines(
+  relations: RelationSnippet[]
+): string[] {
+  const FILTERED = "[content filtered for security]";
+  const out: string[] = [];
+  for (const r of relations) {
+    const src = sanitizeSnippet(r.sourceTitle);
+    const tgt = sanitizeSnippet(r.targetTitle);
+    if (src === FILTERED || tgt === FILTERED) continue;
+    out.push(`- ${src} --[${r.relationType}]--> ${tgt}`);
+  }
+  return out;
+}
+
+/**
+ * Assemble the inner body of <vault-context>: always instruction + facts,
+ * optionally relationships when at least one line fits in the remaining
+ * budget. Relationships are the first thing dropped — if the relationships
+ * XML wrapper alone would exceed `remainingBudgetTokens`, the whole block
+ * is omitted rather than emitting an empty wrapper.
+ */
+export function buildVaultContextInner(
+  factsBlock: string,
+  relations: RelationSnippet[],
+  remainingBudgetTokens: number
+): string {
+  const lines: string[] = [];
+  lines.push(INSTRUCTION_XML);
+  lines.push(`<facts>\n${factsBlock}\n</facts>`);
+
+  if (relations.length === 0 || remainingBudgetTokens <= 0) {
+    return lines.join("\n");
+  }
+
+  const relationLines = renderRelationshipLines(relations);
+  if (relationLines.length === 0) return lines.join("\n");
+
+  // The XML wrapper itself consumes tokens — if there's no room for even one
+  // line on top of the wrapper, drop the block entirely.
+  const fittedLines: string[] = [];
+  let used = RELATIONSHIPS_XML_OVERHEAD_TOKENS;
+  for (const line of relationLines) {
+    const lineTokens = estimateTokens(line + "\n");
+    if (used + lineTokens > remainingBudgetTokens) break;
+    fittedLines.push(line);
+    used += lineTokens;
+  }
+  if (fittedLines.length === 0) return lines.join("\n");
+
+  lines.push(`<relationships>\n${fittedLines.join("\n")}\n</relationships>`);
+  return lines.join("\n");
+}
+
 /**
  * Check if the agent should be nudged to use lifecycle tools.
  * Returns true if N+ context-surfacing invocations have occurred since the

package/src/hooks/feedback-loop.ts

@@ -10,12 +10,18 @@
  */
 
 import type { Store } from "../store.ts";
+import { resolveStore } from "../store.ts";
+import { listVaults } from "../config.ts";
 import type { HookInput, HookOutput } from "../hooks.ts";
 import {
   makeEmptyOutput,
   readTranscript,
   validateTranscriptPath,
 } from "../hooks.ts";
+import {
+  segmentTranscriptIntoTurns,
+  attributeRecallReferences,
+} from "../recall-attribution.ts";
 
 // =============================================================================
 // Handler
@@ -129,6 +135,33 @@ export async function feedbackLoop(
     // Non-critical — don't block feedback loop on utility tracking errors
   }
 
+  // Recall tracking: per-turn attribution using transcript segmentation.
+  // Reads full transcript, segments into turns, zips with context_usage rows,
+  // checks references per-turn rather than session-globally.
+  try {
+    const allMessages = readTranscript(transcriptPath, 500);
+    const turns = segmentTranscriptIntoTurns(allMessages);
+    const usages = store.getUsageForSession(sessionId);
+
+    // General vault attribution
+    attributeRecallReferences(store, sessionId, usages, turns);
+
+    // Cross-vault: attribute recall events in any configured named vaults.
+    // Each vault has its own context_usage rows (mirrored during context-surfacing).
+    const vaultNames = listVaults();
+    for (const vaultName of vaultNames) {
+      try {
+        const vaultStore = resolveStore(vaultName);
+        const vaultUsages = vaultStore.getUsageForSession(sessionId);
+        if (vaultUsages.length > 0) {
+          attributeRecallReferences(vaultStore, sessionId, vaultUsages, turns);
+        }
+      } catch { /* vault unavailable — skip */ }
+    }
+  } catch {
+    // Non-critical — don't block feedback loop on recall tracking errors
+  }
+
   // Silent return — feedback loop doesn't inject context
   return makeEmptyOutput("feedback-loop");
 }
@@ -195,6 +228,13 @@ function trackUtilitySignals(
 // Reference Detection
 // =============================================================================
 
+// Recall attribution logic is in src/recall-attribution.ts
+// (attributeRecallReferences, segmentTranscriptIntoTurns)
+
+// =============================================================================
+// Reference Detection
+// =============================================================================
+
 function checkTitleReference(store: Store, path: string, text: string): boolean {
   try {
     const parts = path.split("/");

package/src/hooks.ts

@@ -385,23 +385,28 @@ export function logInjection(
   sessionId: string,
   hookName: string,
   injectedPaths: string[],
-  estimatedTokens: number
-): void {
+  estimatedTokens: number,
+  turnIndex?: number
+): number {
   try {
-    store.insertUsage({
+    const usageId = store.insertUsage({
       sessionId,
       timestamp: new Date().toISOString(),
       hookName,
       injectedPaths,
       estimatedTokens,
       wasReferenced: 0,
+      turnIndex,
     });
 
     // Record co-activation for all injected paths (E3)
     if (injectedPaths.length >= 2) {
       store.recordCoActivation(injectedPaths);
     }
+
+    return usageId;
   } catch {
     // Non-fatal: don't crash hook if usage logging fails
+    return -1;
   }
 }

package/src/mcp.ts

@@ -2277,6 +2277,11 @@ This is the recommended entry point for ALL memory queries.`,
       const config = loadConfig();
       const policy = config.lifecycle;
 
+      // Recall tracking summary
+      const recallStats = store.getRecallStatsAll(1);
+      const highDiversity = recallStats.filter(r => r.diversityScore >= 0.4 && r.spacingScore >= 0.5 && r.recallCount >= 3);
+      const highNoise = recallStats.filter(r => r.recallCount >= 5 && r.negativeCount > r.recallCount * 0.8);
+
       const lines = [
         `Active: ${stats.active}`,
         `Archived (auto): ${stats.archived}`,
@@ -2286,6 +2291,10 @@ This is the recommended entry point for ALL memory queries.`,
         `Never accessed: ${stats.neverAccessed}`,
         `Oldest access: ${stats.oldestAccess?.slice(0, 10) || "n/a"}`,
         "",
+        `Recall tracking: ${recallStats.length} docs tracked`,
+        `  Pin candidates (high diversity+spacing): ${highDiversity.length}`,
+        `  Snooze candidates (surfaced often, rarely referenced): ${highNoise.length}`,
+        "",
         `Policy: ${policy ? `archive after ${policy.archive_after_days}d, purge after ${policy.purge_after_days ?? "never"}, dry_run=${policy.dry_run}` : "none configured"}`,
       ];
 
@@ -2322,7 +2331,29 @@ This is the recommended entry point for ALL memory queries.`,
         const lines = candidates.map(c =>
           `- ${c.collection}/${c.path} (${c.content_type}, modified ${c.modified_at.slice(0, 10)}, accessed ${c.last_accessed_at?.slice(0, 10) || "never"})`
         );
-        return { content: [{ type: "text", text: `Would archive ${candidates.length} document(s):\n${lines.join("\n") || "(none)"}` }] };
+
+        // Recall-based recommendations
+        const recallStats = store.getRecallStatsAll(3);
+        const pinCandidates = recallStats.filter(r => r.diversityScore >= 0.4 && r.spacingScore >= 0.5 && r.recallCount >= 3);
+        const snoozeCandidates = recallStats.filter(r => r.recallCount >= 5 && r.negativeCount > r.recallCount * 0.8);
+
+        const recallLines: string[] = [];
+        if (pinCandidates.length > 0) {
+          recallLines.push("", "Pin candidates (high diversity, multi-day spread, recall≥3):");
+          for (const r of pinCandidates.slice(0, 5)) {
+            const label = r.collection && r.path ? `${r.collection}/${r.path}` : `doc#${r.docId}`;
+            recallLines.push(`  - ${label} (recalls=${r.recallCount}, queries=${r.uniqueQueries}, days=${r.recallDays}, diversity=${r.diversityScore.toFixed(2)}, spacing=${r.spacingScore.toFixed(2)})`);
+          }
+        }
+        if (snoozeCandidates.length > 0) {
+          recallLines.push("", "Snooze candidates (surfaced often, rarely referenced):");
+          for (const r of snoozeCandidates.slice(0, 5)) {
+            const label = r.collection && r.path ? `${r.collection}/${r.path}` : `doc#${r.docId}`;
+            recallLines.push(`  - ${label} (recalls=${r.recallCount}, referenced=${r.recallCount - r.negativeCount}, noise_ratio=${(r.negativeCount / r.recallCount * 100).toFixed(0)}%)`);
+          }
+        }
+
+        return { content: [{ type: "text", text: `Would archive ${candidates.length} document(s):\n${lines.join("\n") || "(none)"}${recallLines.join("\n")}` }] };
       }
 
       const archived = store.archiveDocuments(candidates.map(c => c.id));