clawmem 0.1.0

package/src/hooks.ts

@@ -0,0 +1,367 @@
+/**
+ * ClawMem Hook Runner - stdin/stdout JSON hook dispatch for Claude Code
+ *
+ * Claude Code hooks send JSON on stdin and expect JSON on stdout.
+ * This module provides the I/O layer and dispatches to individual hook handlers.
+ */
+
+import type { Store } from "./store.ts";
+import { createHash } from "node:crypto";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export type HookInput = {
+  sessionId?: string;
+  prompt?: string;
+  transcriptPath?: string;
+  hookEventName?: string;
+  toolInput?: Record<string, unknown>;
+};
+
+export type HookOutput = {
+  continue?: boolean;
+  suppressOutput?: boolean;
+  stopReason?: string;
+  decision?: "approve" | "block";
+  reason?: string;
+  systemMessage?: string;
+  permissionDecision?: "allow" | "deny" | "ask";
+  hookSpecificOutput?: {
+    hookEventName?: string;
+    additionalContext?: string;
+  };
+};
+
+// =============================================================================
+// I/O
+// =============================================================================
+
+/**
+ * Read hook input from stdin (Claude Code sends JSON with snake_case keys).
+ * Maps snake_case → camelCase to match HookInput type.
+ */
+export async function readHookInput(): Promise<HookInput> {
+  const chunks: Uint8Array[] = [];
+  for await (const chunk of Bun.stdin.stream()) {
+    chunks.push(chunk);
+  }
+  const raw = Buffer.concat(chunks).toString("utf-8").trim();
+  if (!raw) return {};
+  try {
+    const parsed = JSON.parse(raw);
+    return {
+      sessionId: parsed.session_id ?? parsed.sessionId,
+      prompt: parsed.prompt,
+      transcriptPath: parsed.transcript_path ?? parsed.transcriptPath,
+      hookEventName: parsed.hook_event_name ?? parsed.hookEventName,
+      toolInput: parsed.tool_input ?? parsed.toolInput,
+    };
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Write hook output to stdout (Claude Code reads JSON).
+ */
+export function writeHookOutput(output: HookOutput): void {
+  console.log(JSON.stringify(output));
+}
+
+/**
+ * Map internal hook names → Claude Code event names for hookSpecificOutput.
+ * Only UserPromptSubmit and PostToolUse support additionalContext.
+ * Stop/SessionStart hooks must NOT include hookSpecificOutput.
+ */
+const HOOK_EVENT_MAP: Record<string, string | null> = {
+  "context-surfacing": "UserPromptSubmit",
+  "session-bootstrap": null,       // SessionStart — no hookSpecificOutput
+  "staleness-check": null,         // SessionStart — no hookSpecificOutput
+  "decision-extractor": null,      // Stop — no hookSpecificOutput
+  "handoff-generator": null,       // Stop — no hookSpecificOutput
+  "feedback-loop": null,           // Stop — no hookSpecificOutput
+  "precompact-extract": null,      // PreCompact — side-effect only, no context injection
+  "postcompact-inject": "SessionStart", // SessionStart(compact) — injects additionalContext
+  "pretool-inject": null,          // PreToolUse — disabled (cannot inject additionalContext; E13 folded into context-surfacing)
+};
+
+/**
+ * Create a successful output with additional context injected into Claude's prompt.
+ */
+export function makeContextOutput(
+  hookName: string,
+  context: string
+): HookOutput {
+  const eventName = HOOK_EVENT_MAP[hookName];
+  if (!eventName) {
+    // Stop/SessionStart hooks don't support hookSpecificOutput
+    return { continue: true, suppressOutput: false };
+  }
+  return {
+    continue: true,
+    suppressOutput: false,
+    hookSpecificOutput: {
+      hookEventName: eventName,
+      additionalContext: context,
+    },
+  };
+}
+
+/**
+ * Create an empty output (no context to inject).
+ */
+export function makeEmptyOutput(hookName?: string): HookOutput {
+  const eventName = hookName ? HOOK_EVENT_MAP[hookName] : undefined;
+  if (hookName && !eventName) {
+    // Stop/SessionStart hooks don't support hookSpecificOutput
+    return { continue: true, suppressOutput: false };
+  }
+  return {
+    continue: true,
+    suppressOutput: false,
+    ...(eventName && {
+      hookSpecificOutput: {
+        hookEventName: eventName,
+        additionalContext: "",
+      },
+    }),
+  };
+}
+
+// =============================================================================
+// Token Estimation
+// =============================================================================
+
+/**
+ * Estimate token count (~4 chars per token).
+ */
+export function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+// =============================================================================
+// Heartbeat / Dedupe Suppression (IO4)
+// =============================================================================
+
+const DEFAULT_HEARTBEAT_SUBSTRINGS = [
+  "heartbeat",
+  "health check",
+  "keepalive",
+  "keep-alive",
+  "status check",
+  "are you alive",
+  "still alive",
+  "ping",
+  "pong",
+];
+
+function getHeartbeatSubstrings(): string[] {
+  const raw = (Bun.env.CLAWMEM_HEARTBEAT_PATTERNS || "").trim();
+  const extra = raw
+    ? raw.split(",").map(s => s.trim().toLowerCase()).filter(Boolean)
+    : [];
+  // Deduplicate while preserving order.
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const s of [...DEFAULT_HEARTBEAT_SUBSTRINGS, ...extra]) {
+    if (!seen.has(s)) {
+      seen.add(s);
+      out.push(s);
+    }
+  }
+  return out;
+}
+
+export function isHeartbeatPrompt(prompt: string): boolean {
+  if (Bun.env.CLAWMEM_DISABLE_HEARTBEAT_SUPPRESSION === "true") return false;
+  const p = (prompt || "").trim().toLowerCase();
+  if (!p) return true;
+  if (p.startsWith("/")) return true;
+
+  // Exact tiny pings.
+  if (p === "ping" || p === "pong" || p === "heartbeat") return true;
+
+  const subs = getHeartbeatSubstrings();
+  return subs.some(s => p.includes(s));
+}
+
+export function wasPromptSeenRecently(store: Store, hookName: string, prompt: string): boolean {
+  const windowSecRaw = (Bun.env.CLAWMEM_HOOK_DEDUP_WINDOW_SEC || "").trim();
+  const windowSec = windowSecRaw ? parseInt(windowSecRaw, 10) : 600;
+  if (!Number.isFinite(windowSec) || windowSec <= 0) return false;
+
+  const normalized = (prompt || "").trim();
+  if (!normalized) return false;
+
+  const hash = createHash("sha256").update(normalized).digest("hex");
+  const now = new Date();
+  const nowIso = now.toISOString();
+
+  const row = store.db
+    .prepare("SELECT last_seen_at FROM hook_dedupe WHERE hook_name = ? AND prompt_hash = ? LIMIT 1")
+    .get(hookName, hash) as { last_seen_at: string } | null;
+
+  let recent = false;
+  if (row?.last_seen_at) {
+    const lastMs = Date.parse(row.last_seen_at);
+    if (!Number.isNaN(lastMs)) {
+      recent = (now.getTime() - lastMs) < windowSec * 1000;
+    }
+  }
+
+  const preview = normalized.slice(0, 120);
+  store.db.prepare(`
+    INSERT INTO hook_dedupe (hook_name, prompt_hash, prompt_preview, last_seen_at)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT(hook_name, prompt_hash) DO UPDATE SET
+      prompt_preview = excluded.prompt_preview,
+      last_seen_at = excluded.last_seen_at
+  `).run(hookName, hash, preview, nowIso);
+
+  return recent;
+}
+
+// =============================================================================
+// Transcript Parsing
+// =============================================================================
+
+export type TranscriptMessage = {
+  role: "user" | "assistant" | "system";
+  content: string;
+};
+
+/**
+ * Read and parse a Claude Code transcript (.jsonl file).
+ * Returns the last N messages.
+ */
+export function readTranscript(
+  transcriptPath: string,
+  lastN: number = 200,
+  roleFilter?: "user" | "assistant"
+): TranscriptMessage[] {
+  try {
+    const content = require("fs").readFileSync(transcriptPath, "utf-8");
+    const lines = content.split("\n").filter((l: string) => l.trim());
+    const messages: TranscriptMessage[] = [];
+
+    for (const line of lines) {
+      try {
+        const entry = JSON.parse(line);
+        // Claude Code transcript: {type, message: {role, content}} or flat {role, content}
+        const msg = entry.message ?? entry;
+        if (msg.role && msg.content) {
+          const role = msg.role as TranscriptMessage["role"];
+          const text = typeof msg.content === "string"
+            ? msg.content
+            : Array.isArray(msg.content)
+              ? msg.content
+                  .map((b: any) => {
+                    if (b.type === "text") return b.text;
+                    if (b.type === "tool_use") return `[tool_use name="${b.name}" id="${b.id}"] ${JSON.stringify(b.input ?? {})}`;
+                    if (b.type === "tool_result") return `[tool_result id="${b.tool_use_id}"] ${typeof b.content === "string" ? b.content.slice(0, 500) : ""}`;
+                    return "";
+                  })
+                  .filter((s: string) => s)
+                  .join("\n")
+              : JSON.stringify(msg.content);
+
+          if (!roleFilter || role === roleFilter) {
+            messages.push({ role, content: text });
+          }
+        }
+      } catch {
+        // Skip malformed lines
+      }
+    }
+
+    return messages.slice(-lastN);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Validate a transcript path (security: must be absolute, .jsonl, regular file, <50MB).
+ */
+export function validateTranscriptPath(path: string | undefined): string | null {
+  if (!path) return null;
+  if (!require("path").isAbsolute(path)) return null;
+  if (!path.endsWith(".jsonl")) return null;
+
+  try {
+    const stat = require("fs").statSync(path);
+    if (!stat.isFile()) return null;
+    if (stat.size > 50 * 1024 * 1024) return null; // 50MB limit
+    return path;
+  } catch {
+    return null;
+  }
+}
+
+// =============================================================================
+// Snippet Helpers
+// =============================================================================
+
+/**
+ * Smart truncate: break at paragraph → sentence → newline → word boundary.
+ */
+export function smartTruncate(text: string, maxChars: number = 300): string {
+  if (text.length <= maxChars) return text;
+
+  const truncated = text.slice(0, maxChars);
+
+  // Try paragraph break
+  const paraIdx = truncated.lastIndexOf("\n\n");
+  if (paraIdx > maxChars * 0.5) return truncated.slice(0, paraIdx).trimEnd();
+
+  // Try sentence break
+  const sentenceMatch = truncated.match(/^(.+[.!?])\s/s);
+  if (sentenceMatch && sentenceMatch[1]!.length > maxChars * 0.5) {
+    return sentenceMatch[1]!;
+  }
+
+  // Try newline break
+  const nlIdx = truncated.lastIndexOf("\n");
+  if (nlIdx > maxChars * 0.5) return truncated.slice(0, nlIdx).trimEnd();
+
+  // Try word boundary
+  const wordIdx = truncated.lastIndexOf(" ");
+  if (wordIdx > maxChars * 0.5) return truncated.slice(0, wordIdx).trimEnd() + "...";
+
+  return truncated.trimEnd() + "...";
+}
+
+// =============================================================================
+// Logging
+// =============================================================================
+
+/**
+ * Log a context injection to the usage tracking table.
+ */
+export function logInjection(
+  store: Store,
+  sessionId: string,
+  hookName: string,
+  injectedPaths: string[],
+  estimatedTokens: number
+): void {
+  try {
+    store.insertUsage({
+      sessionId,
+      timestamp: new Date().toISOString(),
+      hookName,
+      injectedPaths,
+      estimatedTokens,
+      wasReferenced: 0,
+    });
+
+    // Record co-activation for all injected paths (E3)
+    if (injectedPaths.length >= 2) {
+      store.recordCoActivation(injectedPaths);
+    }
+  } catch {
+    // Non-fatal: don't crash hook if usage logging fails
+  }
+}

package/src/indexer.ts

@@ -0,0 +1,327 @@
+/**
+ * ClawMem Indexer - Vault walking, frontmatter parsing, SAME metadata extraction
+ *
+ * Combines QMD's document indexing with SAME's frontmatter metadata system.
+ */
+
+import { Glob } from "bun";
+import { readFileSync, statSync } from "fs";
+import { basename, relative } from "path";
+import matter from "gray-matter";
+import { createHash } from "crypto";
+import type { Store } from "./store.ts";
+import { inferContentType, confidenceScore, type ContentType } from "./memory.ts";
+import { getDefaultLlamaCpp } from "./llm.ts";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export interface DocumentMeta {
+  title?: string;
+  tags?: string[];
+  domain?: string;
+  workstream?: string;
+  content_type?: ContentType;
+  review_by?: string;
+}
+
+export interface IndexStats {
+  added: number;
+  updated: number;
+  unchanged: number;
+  removed: number;
+}
+
+// =============================================================================
+// Exclusion Rules
+// =============================================================================
+
+const EXCLUDED_DIRS = new Set([
+  "_PRIVATE",
+  ".clawmem",
+  ".git",
+  ".obsidian",
+  ".logseq",
+  ".foam",
+  ".dendron",
+  ".trash",
+  ".stversions",
+  "node_modules",
+  ".cache",
+  "vendor",
+  "dist",
+  "build",
+  "gits",
+  "scraped",
+]);
+
+export function shouldExclude(relativePath: string): boolean {
+  const segments = relativePath.split("/");
+  return segments.some(s => EXCLUDED_DIRS.has(s) || (s.startsWith(".") && s !== "."));
+}
+
+// =============================================================================
+// Content Hashing
+// =============================================================================
+
+export function hashContent(content: string): string {
+  return createHash("sha256").update(content, "utf-8").digest("hex");
+}
+
+// =============================================================================
+// Title Extraction
+// =============================================================================
+
+export function extractTitle(content: string, filename: string): string {
+  const lines = content.split("\n");
+  for (const line of lines) {
+    const match = line.match(/^#+\s+(.+)/);
+    if (match?.[1]) return match[1].trim();
+  }
+  return basename(filename).replace(/\.(md|txt)$/i, "");
+}
+
+// =============================================================================
+// Frontmatter Parsing
+// =============================================================================
+
+export function parseDocument(content: string, relativePath: string): { body: string; meta: DocumentMeta } {
+  try {
+    const { data, content: body } = matter(content);
+    return {
+      body,
+      meta: {
+        title: data.title as string | undefined,
+        tags: Array.isArray(data.tags) ? data.tags.map(String) : undefined,
+        domain: data.domain as string | undefined,
+        workstream: data.workstream as string | undefined,
+        content_type: (data.content_type as ContentType) || inferContentType(relativePath),
+        review_by: data.review_by as string | undefined,
+      },
+    };
+  } catch {
+    // If frontmatter parsing fails, treat entire content as body
+    return {
+      body: content,
+      meta: {
+        content_type: inferContentType(relativePath),
+      },
+    };
+  }
+}
+
+// =============================================================================
+// Quality Scoring
+// =============================================================================
+
+export function computeQualityScore(body: string, meta: DocumentMeta): number {
+  let score = 0.3; // Base
+
+  // Length signals
+  if (body.length > 200) score += 0.1;
+  if (body.length > 500) score += 0.1;
+
+  // Structure signals
+  if (/^##\s+/m.test(body)) score += 0.1;
+  if (/^[-*]\s+/m.test(body)) score += 0.05;
+
+  // Decision/commitment keywords
+  if (/\b(decided?\s+to|chose|will\s+use|switching\s+to|going\s+with|selected|adopted)\b/i.test(body)) {
+    score += 0.15;
+  }
+
+  // Correction keywords
+  if (/\b(fix(ed)?|bug|resolved|corrected|patched|root\s+cause)\b/i.test(body)) {
+    score += 0.1;
+  }
+
+  // Frontmatter richness: +0.05 per populated field, max +0.15
+  let metaBonus = 0;
+  if (meta.tags && meta.tags.length > 0) metaBonus += 0.05;
+  if (meta.domain) metaBonus += 0.05;
+  if (meta.workstream) metaBonus += 0.05;
+  score += Math.min(0.15, metaBonus);
+
+  // Penalty for trivial stubs
+  if (body.length < 50) score -= 0.1;
+
+  return Math.max(0, Math.min(1.0, score));
+}
+
+// =============================================================================
+// Collection Indexing
+// =============================================================================
+
+// Expand top-level brace patterns into individual glob strings.
+// e.g. "{MEMORY.md,memory/x.md}" => ["MEMORY.md", "memory/x.md"]
+// Patterns without braces pass through unchanged.
+function expandBraces(pattern: string): string[] {
+  const match = pattern.match(/^\{(.+)\}$/);
+  if (!match) return [pattern];
+  return match[1]!.split(",").map(s => s.trim());
+}
+
+export async function indexCollection(
+  store: Store,
+  collectionName: string,
+  collectionPath: string,
+  pattern: string = "**/*.md"
+): Promise<IndexStats> {
+  const stats: IndexStats = { added: 0, updated: 0, unchanged: 0, removed: 0 };
+  const activePaths = new Set<string>();
+
+  // Get LLM instance for A-MEM enrichment
+  const llm = getDefaultLlamaCpp();
+
+  // Bun.Glob doesn't support brace expansion {a,b,c} — expand manually
+  const patterns = expandBraces(pattern);
+  const seen = new Set<string>();
+  const allEntries: string[] = [];
+  for (const p of patterns) {
+    const glob = new Glob(p);
+    for (const f of glob.scanSync({ cwd: collectionPath, followSymlinks: false, absolute: false })) {
+      if (!seen.has(f)) {
+        seen.add(f);
+        allEntries.push(f);
+      }
+    }
+  }
+
+  // Collect doc IDs that need post-index enrichment (deferred until after commit)
+  const enrichQueue: { docId: number; isNew: boolean }[] = [];
+
+  // Wrap all DB writes in a transaction for atomicity
+  store.db.exec("BEGIN");
+  try {
+    for (const relativePath of allEntries) {
+      if (shouldExclude(relativePath)) continue;
+
+      activePaths.add(relativePath);
+      const absolutePath = `${collectionPath}/${relativePath}`;
+
+      let content: string;
+      let mtime: Date;
+      try {
+        content = readFileSync(absolutePath, "utf-8");
+        mtime = statSync(absolutePath).mtime;
+      } catch {
+        continue; // File may have been deleted between scan and read
+      }
+
+      const contentHash = hashContent(content);
+      const now = new Date().toISOString();
+
+      // Check if document already exists
+      const existing = store.findActiveDocument(collectionName, relativePath);
+
+      if (existing) {
+        // Check if content changed via content hash
+        const existingRow = store.db.prepare(
+          "SELECT content_hash FROM documents WHERE id = ?"
+        ).get(existing.id) as { content_hash: string | null } | null;
+
+        if (existingRow?.content_hash === contentHash) {
+          stats.unchanged++;
+          continue;
+        }
+
+        // Content changed — update
+        const { body, meta } = parseDocument(content, relativePath);
+        const title = meta.title || extractTitle(body, relativePath);
+        const docHash = hashContent(body);
+
+        store.insertContent(docHash, body, now);
+        store.updateDocument(existing.id, title, docHash, mtime.toISOString());
+
+        // Update SAME metadata
+        const contentType = meta.content_type || inferContentType(relativePath);
+        store.updateDocumentMeta(existing.id, {
+          domain: meta.domain,
+          workstream: meta.workstream,
+          tags: meta.tags ? JSON.stringify(meta.tags) : undefined,
+          content_type: contentType,
+          review_by: meta.review_by,
+          confidence: confidenceScore(contentType, mtime, 0),
+          quality_score: computeQualityScore(body, meta),
+        });
+
+        // Update content_hash for next incremental check
+        store.db.prepare("UPDATE documents SET content_hash = ? WHERE id = ?").run(contentHash, existing.id);
+
+        // Defer A-MEM enrichment until after commit
+        enrichQueue.push({ docId: existing.id, isNew: false });
+
+        stats.updated++;
+      } else {
+        // Check for inactive (previously removed) doc at same path — reactivate instead of inserting
+        const inactive = store.db.prepare(
+          "SELECT id, hash FROM documents WHERE collection = ? AND path = ? AND active = 0"
+        ).get(collectionName, relativePath) as { id: number; hash: string } | null;
+
+        const { body, meta } = parseDocument(content, relativePath);
+        const title = meta.title || extractTitle(body, relativePath);
+        const docHash = hashContent(body);
+        const contentType = meta.content_type || inferContentType(relativePath);
+
+        store.insertContent(docHash, body, now);
+
+        if (inactive) {
+          // Reactivate existing row
+          store.db.prepare("UPDATE documents SET active = 1, hash = ?, title = ?, modified_at = ?, content_hash = ? WHERE id = ?")
+            .run(docHash, title, mtime.toISOString(), contentHash, inactive.id);
+          store.updateDocumentMeta(inactive.id, {
+            domain: meta.domain,
+            workstream: meta.workstream,
+            tags: meta.tags ? JSON.stringify(meta.tags) : undefined,
+            content_type: contentType,
+            review_by: meta.review_by,
+            confidence: confidenceScore(contentType, mtime, 0),
+            quality_score: computeQualityScore(body, meta),
+          });
+          enrichQueue.push({ docId: inactive.id, isNew: false });
+        } else {
+          // Truly new document
+          store.insertDocument(collectionName, relativePath, title, docHash, now, mtime.toISOString());
+          const newDoc = store.findActiveDocument(collectionName, relativePath);
+          if (newDoc) {
+            store.updateDocumentMeta(newDoc.id, {
+              domain: meta.domain,
+              workstream: meta.workstream,
+              tags: meta.tags ? JSON.stringify(meta.tags) : undefined,
+              content_type: contentType,
+              review_by: meta.review_by,
+              confidence: confidenceScore(contentType, mtime, 0),
+              quality_score: computeQualityScore(body, meta),
+            });
+            store.db.prepare("UPDATE documents SET content_hash = ? WHERE id = ?").run(contentHash, newDoc.id);
+            enrichQueue.push({ docId: newDoc.id, isNew: true });
+          }
+        }
+
+        stats.added++;
+      }
+    }
+
+    // Deactivate documents that no longer exist on disk
+    const storedPaths = store.getActiveDocumentPaths(collectionName);
+    for (const storedPath of storedPaths) {
+      if (!activePaths.has(storedPath)) {
+        store.deactivateDocument(collectionName, storedPath);
+        stats.removed++;
+      }
+    }
+
+    store.db.exec("COMMIT");
+  } catch (err) {
+    store.db.exec("ROLLBACK");
+    throw err;
+  }
+
+  // A-MEM enrichment runs after successful commit (LLM calls should not block transaction)
+  for (const { docId, isNew } of enrichQueue) {
+    await store.postIndexEnrich(llm, docId, isNew);
+  }
+
+  return stats;
+}