clawmem 0.1.0

package/src/profile.ts

@@ -0,0 +1,346 @@
+/**
+ * User Profile Abstraction - Two-tier profile (static facts + dynamic context)
+ *
+ * Builds a profile document from vault contents:
+ * - Static: persistent facts extracted from decisions, hubs, and notes
+ * - Dynamic: recent context from last sessions and progress docs
+ *
+ * Stored at _clawmem/profile.md, injected at session start.
+ */
+
+import type { Store } from "./store.ts";
+import { hashContent } from "./indexer.ts";
+import { smartTruncate } from "./hooks.ts";
+import { MAX_LEVENSHTEIN_LENGTH } from "./limits.ts";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export type Profile = {
+  static: string[];
+  dynamic: string[];
+  updatedAt: string;
+};
+
+// =============================================================================
+// Config
+// =============================================================================
+
+const STATIC_MAX_TOKENS = 500;
+const DYNAMIC_MAX_TOKENS = 300;
+const STATIC_MAX_FACTS = 30;
+const DYNAMIC_MAX_ITEMS = 10;
+const PROFILE_PATH = "profile.md";
+const PROFILE_COLLECTION = "_clawmem";
+const STALE_SESSION_THRESHOLD = 5;
+
+// =============================================================================
+// Profile Building
+// =============================================================================
+
+export function buildStaticProfile(store: Store): string[] {
+  const facts: string[] = [];
+  const seen = new Set<string>();
+
+  // Extract from decisions
+  const decisions = store.getDocumentsByType("decision", 20);
+  for (const d of decisions) {
+    const body = store.getDocumentBody({
+      filepath: `${d.collection}/${d.path}`,
+      displayPath: `${d.collection}/${d.path}`,
+    } as any);
+    if (!body) continue;
+
+    const bullets = extractBullets(body);
+    for (const bullet of bullets) {
+      const key = bullet.toLowerCase().trim().slice(0, 60);
+      if (seen.has(key)) continue;
+      if (isTooSimilar(key, seen)) continue;
+      seen.add(key);
+      facts.push(bullet);
+    }
+  }
+
+  // Extract from hub documents
+  const hubs = store.getDocumentsByType("hub", 10);
+  for (const h of hubs) {
+    const body = store.getDocumentBody({
+      filepath: `${h.collection}/${h.path}`,
+      displayPath: `${h.collection}/${h.path}`,
+    } as any);
+    if (!body) continue;
+
+    const bullets = extractBullets(body);
+    for (const bullet of bullets) {
+      const key = bullet.toLowerCase().trim().slice(0, 60);
+      if (seen.has(key)) continue;
+      if (isTooSimilar(key, seen)) continue;
+      seen.add(key);
+      facts.push(bullet);
+    }
+  }
+
+  // Truncate to budget
+  const maxChars = STATIC_MAX_TOKENS * 4;
+  let charCount = 0;
+  const result: string[] = [];
+  for (const fact of facts.slice(0, STATIC_MAX_FACTS)) {
+    if (charCount + fact.length > maxChars) break;
+    result.push(fact);
+    charCount += fact.length;
+  }
+
+  return result;
+}
+
+export function buildDynamicProfile(store: Store): string[] {
+  const items: string[] = [];
+
+  // Recent sessions
+  const sessions = store.getRecentSessions(5);
+  for (const s of sessions) {
+    if (!s.handoffPath) continue;
+
+    const body = store.getDocumentBody({
+      filepath: s.handoffPath,
+      displayPath: s.handoffPath,
+    } as any);
+    if (!body) continue;
+
+    // Extract "Current State" and "Next Session Should" sections
+    const currentState = extractSection(body, "Current State");
+    const nextSession = extractSection(body, "Next Session Should");
+
+    if (currentState) {
+      items.push(`Current: ${smartTruncate(currentState, 150)}`);
+    }
+    if (nextSession) {
+      items.push(`Next: ${smartTruncate(nextSession, 150)}`);
+    }
+  }
+
+  // Recent progress documents
+  const cutoff = new Date();
+  cutoff.setDate(cutoff.getDate() - 7);
+  const progress = store.getDocumentsByType("progress", 5);
+  const recent = progress.filter(p => p.modifiedAt >= cutoff.toISOString());
+
+  for (const p of recent) {
+    items.push(`Progress: ${p.title} (${p.modifiedAt.slice(0, 10)})`);
+  }
+
+  // Truncate to budget
+  const maxChars = DYNAMIC_MAX_TOKENS * 4;
+  let charCount = 0;
+  const result: string[] = [];
+  for (const item of items.slice(0, DYNAMIC_MAX_ITEMS)) {
+    if (charCount + item.length > maxChars) break;
+    result.push(item);
+    charCount += item.length;
+  }
+
+  return result;
+}
+
+// =============================================================================
+// Profile Persistence
+// =============================================================================
+
+export function updateProfile(store: Store): void {
+  const staticFacts = buildStaticProfile(store);
+  const dynamicItems = buildDynamicProfile(store);
+  const now = new Date().toISOString();
+
+  const body = formatProfileDocument(staticFacts, dynamicItems);
+  const hash = hashContent(body);
+
+  // Store content
+  store.insertContent(hash, body, now);
+
+  // Upsert document (handle active, inactive, or missing)
+  const existing = store.findActiveDocument(PROFILE_COLLECTION, PROFILE_PATH);
+  if (existing) {
+    store.updateDocument(existing.id, "User Profile", hash, now);
+  } else {
+    // Check for inactive row (UNIQUE(collection, path) prevents re-insert)
+    const inactive = store.findAnyDocument(PROFILE_COLLECTION, PROFILE_PATH);
+    if (inactive) {
+      // Reactivate and update
+      store.reactivateDocument(inactive.id, "User Profile", hash, now);
+      store.updateDocumentMeta(inactive.id, {
+        content_type: "hub",
+        tags: JSON.stringify(["auto-generated", "profile"]),
+      });
+    } else {
+      try {
+        store.insertDocument(PROFILE_COLLECTION, PROFILE_PATH, "User Profile", hash, now, now);
+        const doc = store.findActiveDocument(PROFILE_COLLECTION, PROFILE_PATH);
+        if (doc) {
+          store.updateDocumentMeta(doc.id, {
+            content_type: "hub",
+            tags: JSON.stringify(["auto-generated", "profile"]),
+          });
+        }
+      } catch {
+        // Collection may not exist yet
+      }
+    }
+  }
+}
+
+export function getProfile(store: Store): Profile | null {
+  const doc = store.findActiveDocument(PROFILE_COLLECTION, PROFILE_PATH);
+  if (!doc) return null;
+
+  const body = store.getDocumentBody({
+    filepath: `${PROFILE_COLLECTION}/${PROFILE_PATH}`,
+    displayPath: `${PROFILE_COLLECTION}/${PROFILE_PATH}`,
+  } as any);
+  if (!body) return null;
+
+  return parseProfileDocument(body);
+}
+
+export function isProfileStale(store: Store): boolean {
+  const doc = store.findActiveDocument(PROFILE_COLLECTION, PROFILE_PATH);
+  if (!doc) return true;
+
+  // Check how many sessions since last profile update
+  const sessions = store.getRecentSessions(STALE_SESSION_THRESHOLD + 1);
+  if (sessions.length === 0) return false;
+
+  // Get the profile's modification timestamp from the document row
+  const rows = store.getDocumentsByType("hub", 50);
+  const profileRow = rows.find(r => r.path === PROFILE_PATH && r.collection === PROFILE_COLLECTION);
+  if (!profileRow) return true;
+
+  const profileDate = profileRow.modifiedAt;
+  const sessionsSince = sessions.filter(s => s.startedAt > profileDate);
+  return sessionsSince.length >= STALE_SESSION_THRESHOLD;
+}
+
+// =============================================================================
+// Formatting
+// =============================================================================
+
+function formatProfileDocument(staticFacts: string[], dynamicItems: string[]): string {
+  const lines = [
+    "---",
+    "content_type: hub",
+    "tags: [auto-generated, profile]",
+    "---",
+    "",
+    "# User Profile",
+    "",
+  ];
+
+  if (staticFacts.length > 0) {
+    lines.push("## Known Context", "");
+    for (const fact of staticFacts) {
+      lines.push(`- ${fact}`);
+    }
+    lines.push("");
+  }
+
+  if (dynamicItems.length > 0) {
+    lines.push("## Current Focus", "");
+    for (const item of dynamicItems) {
+      lines.push(`- ${item}`);
+    }
+    lines.push("");
+  }
+
+  return lines.join("\n");
+}
+
+function parseProfileDocument(body: string): Profile {
+  const staticFacts: string[] = [];
+  const dynamicItems: string[] = [];
+  let updatedAt = "";
+
+  let section = "";
+  for (const line of body.split("\n")) {
+    if (line.startsWith("## Known Context")) {
+      section = "static";
+      continue;
+    }
+    if (line.startsWith("## Current Focus")) {
+      section = "dynamic";
+      continue;
+    }
+    if (line.startsWith("## ")) {
+      section = "";
+      continue;
+    }
+
+    const bullet = line.match(/^-\s+(.+)/);
+    if (!bullet?.[1]) continue;
+
+    if (section === "static") staticFacts.push(bullet[1]);
+    else if (section === "dynamic") dynamicItems.push(bullet[1]);
+  }
+
+  return { static: staticFacts, dynamic: dynamicItems, updatedAt };
+}
+
+// =============================================================================
+// Helpers
+// =============================================================================
+
+function extractBullets(body: string): string[] {
+  const bullets: string[] = [];
+  for (const line of body.split("\n")) {
+    const match = line.match(/^[-*]\s+(.{10,200})/);
+    if (match?.[1]) {
+      bullets.push(match[1].trim());
+    }
+  }
+  return bullets;
+}
+
+function extractSection(body: string, sectionName: string): string | null {
+  const regex = new RegExp(
+    `^#{1,3}\\s+${escapeRegex(sectionName)}\\b[^\\n]*\\n([\\s\\S]*?)(?=^#{1,3}\\s|$)`,
+    "mi"
+  );
+  const match = body.match(regex);
+  if (!match?.[1]) return null;
+  const text = match[1].trim();
+  return text.length > 10 ? text : null;
+}
+
+function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function isTooSimilar(key: string, existing: Set<string>): boolean {
+  for (const e of existing) {
+    if (levenshteinDistance(key, e) < 5) return true;
+  }
+  return false;
+}
+
+function levenshteinDistance(a: string, b: string): number {
+  // Bound inputs to prevent O(n²) memory blowup
+  if (a.length > MAX_LEVENSHTEIN_LENGTH || b.length > MAX_LEVENSHTEIN_LENGTH) return Math.abs(a.length - b.length);
+  if (a.length === 0) return b.length;
+  if (b.length === 0) return a.length;
+
+  const matrix: number[][] = [];
+  for (let i = 0; i <= b.length; i++) matrix[i] = [i];
+  for (let j = 0; j <= a.length; j++) matrix[0]![j] = j;
+
+  for (let i = 1; i <= b.length; i++) {
+    for (let j = 1; j <= a.length; j++) {
+      const cost = b[i - 1] === a[j - 1] ? 0 : 1;
+      matrix[i]![j] = Math.min(
+        matrix[i - 1]![j]! + 1,
+        matrix[i]![j - 1]! + 1,
+        matrix[i - 1]![j - 1]! + cost
+      );
+    }
+  }
+
+  return matrix[b.length]![a.length]!;
+}

package/src/promptguard.ts

@@ -0,0 +1,218 @@
+/**
+ * ClawMem Prompt Injection Guard
+ *
+ * Multi-layer detection system ported from SAME's go-promptguard integration.
+ * Checks vault content for prompt injection attempts before context injection.
+ * Pure pattern-based (no LLM) for sub-ms latency.
+ */
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export interface DetectionResult {
+  safe: boolean;
+  detector: string | null;
+  score: number; // 0.0 = safe, 1.0 = definite injection
+}
+
+// =============================================================================
+// Detection Layers
+// =============================================================================
+
+/**
+ * Layer 1: Legacy string patterns from SAME (13 patterns).
+ * Case-insensitive substring match. Score: 1.0 on match.
+ */
+const LEGACY_PATTERNS = [
+  "ignore previous",
+  "ignore all previous",
+  "ignore above",
+  "disregard previous",
+  "disregard all previous",
+  "you are now",
+  "new instructions",
+  "system prompt",
+  "<system>",
+  "</system>",
+  "IMPORTANT:",
+  "CRITICAL:",
+  "override",
+];
+
+/**
+ * Layer 2: Role injection patterns. Score: 0.9 on match.
+ */
+const ROLE_INJECTION_PATTERNS = [
+  /you are (?:now |a |an |the )/i,
+  /act as (?:a |an |the |if )/i,
+  /pretend (?:you(?:'re| are) |to be )/i,
+  /(?:switch|change) (?:to |into )(?:a |an |the )?(?:new |different )?(?:role|mode|persona)/i,
+  /your (?:new |real |true )(?:role|purpose|function|task)/i,
+];
+
+/**
+ * Layer 3: Instruction override patterns. Score: 0.85 on match.
+ */
+const INSTRUCTION_OVERRIDE_PATTERNS = [
+  /(?:ignore|forget|discard|disregard) (?:all |any )?(?:previous|prior|above|earlier)/i,
+  /(?:new|updated|revised|real) (?:instructions?|directives?|rules?|guidelines?)/i,
+  /(?:do not|don't|never) (?:follow|obey|listen to|adhere to)/i,
+  /(?:bypass|circumvent|override|skip) (?:the |any |all )?(?:rules?|restrictions?|guidelines?|filters?|safety)/i,
+];
+
+/**
+ * Layer 4: Delimiter injection patterns. Score: 0.8 on match.
+ */
+const DELIMITER_PATTERNS = [
+  /<\/?(?:system|user|assistant|human|ai|bot|prompt|instruction)>/i,
+  /\[(?:SYSTEM|INST|\/INST|SYS)\]/i,
+  /```(?:system|instructions?|prompt)\s*\n/i,
+  /={3,}(?:SYSTEM|PROMPT|INSTRUCTIONS?)={3,}/i,
+];
+
+/**
+ * Layer 5: Unicode obfuscation detection. Score: 0.7 on match.
+ */
+const ZERO_WIDTH_CHARS = /[\u200B\u200C\u200D\uFEFF\u00AD\u2060\u2061\u2062\u2063\u2064]/;
+
+// Cyrillic characters that look like Latin
+const CYRILLIC_LOOKALIKES = /[\u0400-\u04FF]/;
+// Greek characters that look like Latin
+const GREEK_LOOKALIKES = /[\u0370-\u03FF]/;
+
+// =============================================================================
+// Detection Functions
+// =============================================================================
+
+/**
+ * Multi-layer prompt injection detection.
+ * Checks layers in order, short-circuits on first match.
+ * Default threshold: 0.6 (same as SAME's go-promptguard config).
+ */
+export function detectInjection(text: string, threshold: number = 0.6): DetectionResult {
+  if (!text || text.length === 0) {
+    return { safe: true, detector: null, score: 0 };
+  }
+
+  // Cap input length for performance
+  const input = text.slice(0, 2000);
+  const lower = input.toLowerCase();
+
+  // Layer 1: Legacy string patterns
+  for (const pattern of LEGACY_PATTERNS) {
+    if (lower.includes(pattern.toLowerCase())) {
+      const result = { safe: false, detector: "legacy_pattern", score: 1.0 };
+      return result.score >= threshold ? result : { safe: true, detector: null, score: result.score };
+    }
+  }
+
+  // Layer 2: Role injection
+  for (const pattern of ROLE_INJECTION_PATTERNS) {
+    if (pattern.test(input)) {
+      const result = { safe: false, detector: "role_injection", score: 0.9 };
+      return result.score >= threshold ? result : { safe: true, detector: null, score: result.score };
+    }
+  }
+
+  // Layer 3: Instruction override
+  for (const pattern of INSTRUCTION_OVERRIDE_PATTERNS) {
+    if (pattern.test(input)) {
+      const result = { safe: false, detector: "instruction_override", score: 0.85 };
+      return result.score >= threshold ? result : { safe: true, detector: null, score: result.score };
+    }
+  }
+
+  // Layer 4: Delimiter injection
+  for (const pattern of DELIMITER_PATTERNS) {
+    if (pattern.test(input)) {
+      const result = { safe: false, detector: "delimiter_injection", score: 0.8 };
+      return result.score >= threshold ? result : { safe: true, detector: null, score: result.score };
+    }
+  }
+
+  // Layer 5: Unicode obfuscation
+  if (ZERO_WIDTH_CHARS.test(input)) {
+    const result = { safe: false, detector: "unicode_obfuscation", score: 0.7 };
+    return result.score >= threshold ? result : { safe: true, detector: null, score: result.score };
+  }
+
+  // Check for mixed scripts (Latin + Cyrillic/Greek in same word — homoglyph attack)
+  if (hasMixedScripts(input)) {
+    const result = { safe: false, detector: "homoglyph", score: 0.7 };
+    return result.score >= threshold ? result : { safe: true, detector: null, score: result.score };
+  }
+
+  // Check normalization deviation
+  if (hasNormalizationDeviation(input)) {
+    const result = { safe: false, detector: "normalization", score: 0.7 };
+    return result.score >= threshold ? result : { safe: true, detector: null, score: result.score };
+  }
+
+  return { safe: true, detector: null, score: 0 };
+}
+
+/**
+ * Sanitize a snippet for safe injection into context.
+ * Returns the original text if safe, or a placeholder if injection detected.
+ */
+export function sanitizeSnippet(text: string, threshold: number = 0.6): string {
+  const result = detectInjection(text, threshold);
+  if (!result.safe) {
+    return "[content filtered for security]";
+  }
+  return text;
+}
+
+// =============================================================================
+// Helpers
+// =============================================================================
+
+/**
+ * Check for mixed Latin + Cyrillic/Greek within individual words.
+ * This detects homoglyph attacks where Cyrillic 'а' replaces Latin 'a'.
+ */
+function hasMixedScripts(text: string): boolean {
+  // Only check if both scripts are present at all
+  const hasLatin = /[a-zA-Z]/.test(text);
+  const hasCyrillic = CYRILLIC_LOOKALIKES.test(text);
+  const hasGreek = GREEK_LOOKALIKES.test(text);
+
+  if (!hasLatin || (!hasCyrillic && !hasGreek)) return false;
+
+  // Check individual words for mixed scripts
+  const words = text.split(/\s+/);
+  for (const word of words) {
+    if (word.length < 3) continue;
+    const wordHasLatin = /[a-zA-Z]/.test(word);
+    const wordHasCyrillic = CYRILLIC_LOOKALIKES.test(word);
+    const wordHasGreek = GREEK_LOOKALIKES.test(word);
+
+    if (wordHasLatin && (wordHasCyrillic || wordHasGreek)) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Check if NFKD normalization changes the text significantly.
+ * Catches confusable characters and encoding tricks.
+ */
+function hasNormalizationDeviation(text: string): boolean {
+  const normalized = text.normalize('NFKD');
+  if (normalized === text) return false;
+
+  // Count character changes — small diacritic changes are fine,
+  // significant changes suggest obfuscation
+  let changes = 0;
+  const minLen = Math.min(text.length, normalized.length);
+  for (let i = 0; i < minLen; i++) {
+    if (text[i] !== normalized[i]) changes++;
+  }
+  changes += Math.abs(text.length - normalized.length);
+
+  // Flag if >5% of characters changed (threshold to avoid false positives on accented text)
+  return changes / text.length > 0.05;
+}

package/src/retrieval-gate.ts

@@ -0,0 +1,106 @@
+/**
+ * Retrieval Gate — Adaptive prompt filtering for context-surfacing
+ *
+ * Determines whether a prompt warrants memory retrieval. Skips greetings,
+ * shell commands, affirmations, pure emoji, and system pings. Forces
+ * retrieval for memory-intent queries even if short.
+ *
+ * Ported from memory-lancedb-pro's adaptive-retrieval.ts + noise-filter.ts,
+ * complementing ClawMem's existing short-prompt, slash-command, heartbeat,
+ * and dedupe gates in context-surfacing.
+ */
+
+// Prompts that should skip retrieval entirely
+const SKIP_PATTERNS = [
+  // Greetings & pleasantries
+  /^(hi|hello|hey|good\s*(morning|afternoon|evening|night)|greetings|yo|sup|howdy|what'?s up)\b/i,
+  // Shell/dev commands (slash commands handled separately in context-surfacing)
+  /^(run|build|test|ls|cd|git|npm|pip|docker|curl|cat|grep|find|make|sudo|bun|node|deno)\b/i,
+  // Simple affirmations/negations
+  /^(yes|no|yep|nope|ok|okay|sure|fine|thanks|thank you|thx|ty|got it|understood|cool|nice|great|good|perfect|awesome)\s*[.!]?$/i,
+  // Continuation prompts
+  /^(go ahead|continue|proceed|do it|start|begin|next)\s*[.!]?$/i,
+  // Pure emoji
+  /^[\p{Emoji}\s]+$/u,
+  // Single-word utility pings
+  /^(ping|pong|test|debug)\s*[.!?]?$/i,
+];
+
+// Prompts that MUST trigger retrieval even if short (checked before skip)
+const FORCE_RETRIEVE_PATTERNS = [
+  /\b(remember|recall|forgot|memory|memories)\b/i,
+  /\b(last time|before|previously|earlier|yesterday|ago)\b/i,
+  /\b(my (name|email|phone|address|birthday|preference))\b/i,
+  /\b(what did (i|we)|did i (tell|say|mention))\b/i,
+];
+
+/**
+ * Normalize OpenClaw-injected metadata from prompts.
+ * Strips cron wrappers, timestamp prefixes, and conversation metadata.
+ */
+function normalizePrompt(prompt: string): string {
+  let s = prompt.trim();
+  // Strip OpenClaw metadata headers
+  s = s.replace(/^(Conversation info|Sender) \(untrusted metadata\):[\s\S]*?\n\s*\n/gim, "");
+  // Strip cron wrapper prefix
+  s = s.trim().replace(/^\[cron:[^\]]+\]\s*/i, "");
+  // Strip timestamp prefix
+  s = s.trim().replace(/^\[[A-Za-z]{3}\s\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}\s[^\]]+\]\s*/, "");
+  return s.trim();
+}
+
+/**
+ * Check if a prompt should skip memory retrieval.
+ * Returns true if retrieval should be skipped.
+ *
+ * This complements (does NOT replace) existing gates in context-surfacing:
+ * - MIN_PROMPT_LENGTH (<20 chars)
+ * - Slash commands (starts with /)
+ * - Heartbeat suppression
+ * - Duplicate prompt dedupe
+ */
+export function shouldSkipRetrieval(prompt: string): boolean {
+  const trimmed = normalizePrompt(prompt);
+
+  // Force retrieve if query has memory-related intent (before length/pattern checks)
+  if (FORCE_RETRIEVE_PATTERNS.some(p => p.test(trimmed))) return false;
+
+  // Too short to be meaningful (below context-surfacing's MIN_PROMPT_LENGTH)
+  if (trimmed.length < 5) return true;
+
+  // Skip if matches any skip pattern
+  if (SKIP_PATTERNS.some(p => p.test(trimmed))) return true;
+
+  // Skip very short non-question messages
+  // CJK characters carry more meaning per character — lower threshold
+  const hasCJK = /[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/.test(trimmed);
+  const minLength = hasCJK ? 6 : 15;
+  if (trimmed.length < minLength && !trimmed.includes('?') && !trimmed.includes('\uff1f')) return true;
+
+  return false;
+}
+
+// =============================================================================
+// Noise Filter — Post-retrieval result filtering
+// =============================================================================
+
+// Agent denial patterns (filter from retrieved results)
+const DENIAL_PATTERNS = [
+  /i don'?t have (any )?(information|data|memory|record)/i,
+  /i'?m not sure about/i,
+  /i don'?t recall/i,
+  /i don'?t remember/i,
+  /no (relevant )?memories found/i,
+  /i don'?t have access to/i,
+];
+
+/**
+ * Check if a retrieved memory snippet is noise that should be filtered.
+ * Use on search results before injection, NOT on indexed documents.
+ */
+export function isRetrievedNoise(text: string): boolean {
+  const trimmed = text.trim();
+  if (trimmed.length < 10) return true;
+  if (DENIAL_PATTERNS.some(p => p.test(trimmed))) return true;
+  return false;
+}