clawmem 0.2.3 → 0.2.5

package/AGENTS.md

@@ -636,6 +636,18 @@ Symptom: "UserPromptSubmit hook error" on context-surfacing hook (intermittent)
   → Default hook timeout is 8s (since v0.1.1). If you have an older install, re-run
     `clawmem setup hooks`. If persistent, restart the watcher: `systemctl --user restart
     clawmem-watcher.service`. Healthy memory is under 100MB — if 400MB+, restart clears it.
+  → v0.2.4 fix: hook's SQLite busy_timeout was 500ms — too tight. During A-MEM enrichment
+    or heavy indexing, watcher write locks exceed 500ms, causing SQLITE_BUSY. Raised to
+    5000ms (matches MCP server). Still completes within the 8s outer timeout.
+
+Symptom: WSL hangs or becomes unresponsive during long sessions / watcher has 100K+ FDs
+  → Pre-v0.2.3: fs.watch(recursive: true) registered inotify watches on EVERY subdirectory,
+    including excluded dirs (gits/, node_modules/, .git/). Broad collection paths like
+    ~/Projects with 67K subdirs exhausted inotify limits.
+  → v0.2.3 fix: watcher walks dir trees at startup, skips excluded subtrees, watches
+    non-excluded dirs individually. 500-dir cap per collection path.
+  → Diagnosis: `ls /proc/$(pgrep -f "clawmem.*watch")/fd | wc -l` — healthy < 15K.
+  → If still high: narrow broad collection paths. See docs/troubleshooting.md for details.
 ```
 
 ## CLI Reference

package/CLAUDE.md

@@ -636,6 +636,18 @@ Symptom: "UserPromptSubmit hook error" on context-surfacing hook (intermittent)
   → Default hook timeout is 8s (since v0.1.1). If you have an older install, re-run
     `clawmem setup hooks`. If persistent, restart the watcher: `systemctl --user restart
     clawmem-watcher.service`. Healthy memory is under 100MB — if 400MB+, restart clears it.
+  → v0.2.4 fix: hook's SQLite busy_timeout was 500ms — too tight. During A-MEM enrichment
+    or heavy indexing, watcher write locks exceed 500ms, causing SQLITE_BUSY. Raised to
+    5000ms (matches MCP server). Still completes within the 8s outer timeout.
+
+Symptom: WSL hangs or becomes unresponsive during long sessions / watcher has 100K+ FDs
+  → Pre-v0.2.3: fs.watch(recursive: true) registered inotify watches on EVERY subdirectory,
+    including excluded dirs (gits/, node_modules/, .git/). Broad collection paths like
+    ~/Projects with 67K subdirs exhausted inotify limits.
+  → v0.2.3 fix: watcher walks dir trees at startup, skips excluded subtrees, watches
+    non-excluded dirs individually. 500-dir cap per collection path.
+  → Diagnosis: `ls /proc/$(pgrep -f "clawmem.*watch")/fd | wc -l` — healthy < 15K.
+  → If still high: narrow broad collection paths. See docs/troubleshooting.md for details.
 ```
 
 ## CLI Reference

package/SKILL.md

@@ -642,6 +642,18 @@ Symptom: "UserPromptSubmit hook error" on context-surfacing hook (intermittent)
   -> Default hook timeout is 8s (since v0.1.1). If you have an older install, re-run
      `clawmem setup hooks`. If persistent, restart the watcher: `systemctl --user restart
      clawmem-watcher.service`. Healthy memory is under 100MB — if 400MB+, restart clears it.
+  -> v0.2.4 fix: hook's SQLite busy_timeout was 500ms — too tight. During A-MEM enrichment
+     or heavy indexing, watcher write locks exceed 500ms, causing SQLITE_BUSY. Raised to
+     5000ms (matches MCP server). Still completes within the 8s outer timeout.
+
+Symptom: WSL hangs or becomes unresponsive during long sessions / watcher has 100K+ FDs
+  -> Pre-v0.2.3: fs.watch(recursive: true) registered inotify watches on EVERY subdirectory,
+     including excluded dirs (gits/, node_modules/, .git/). Broad collection paths like
+     ~/Projects with 67K subdirs exhausted inotify limits.
+  -> v0.2.3 fix: watcher walks dir trees at startup, skips excluded subtrees, watches
+     non-excluded dirs individually. 500-dir cap per collection path.
+  -> Diagnosis: `ls /proc/$(pgrep -f "clawmem.*watch")/fd | wc -l` — healthy < 15K.
+  -> If still high: narrow broad collection paths. See docs/troubleshooting.md.
 ```
 
 ---

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clawmem",
-  "version": "0.2.3",
+  "version": "0.2.5",
   "description": "On-device context engine and memory for AI agents. Claude Code and OpenClaw. Hooks + MCP server + hybrid RAG search.",
   "type": "module",
   "bin": {

package/src/clawmem.ts

@@ -772,7 +772,7 @@ async function cmdSurface(args: string[]) {
   if (!input) process.exit(0);
 
   // Open store: writable for both (context-surfacing writes dedupe data)
-  const s = createStore(undefined, { busyTimeout: 500 });
+  const s = createStore(undefined, { busyTimeout: 5000 });
 
   try {
     if (isBootstrap) {

package/src/indexer.ts

@@ -37,7 +37,7 @@ export interface IndexStats {
 // Exclusion Rules
 // =============================================================================
 
-const EXCLUDED_DIRS = new Set([
+export const EXCLUDED_DIRS = new Set([
   "_PRIVATE",
   ".clawmem",
   ".git",

package/src/watcher.ts

@@ -1,9 +1,14 @@
 /**
  * ClawMem File Watcher - fs.watch with debounce for incremental reindex
+ *
+ * Walks each directory tree at startup, skipping excluded dirs (gits/,
+ * node_modules/, .git/, etc.), and watches only non-excluded directories.
+ * This prevents inotify FD exhaustion on trees with large cloned repos.
  */
 
-import { watch, type WatchEventType } from "fs";
-import { shouldExclude } from "./indexer.ts";
+import { watch, readdirSync, statSync, type WatchEventType } from "fs";
+import { join, relative } from "path";
+import { shouldExclude, EXCLUDED_DIRS } from "./indexer.ts";
 
 export type WatcherOptions = {
   debounceMs?: number;
@@ -11,6 +16,42 @@ export type WatcherOptions = {
   onError?: (error: Error) => void;
 };
 
+/**
+ * Walk a directory tree, returning only directories that are NOT excluded.
+ * Stops recursion into excluded subtrees (gits/, node_modules/, .git/, etc.).
+ */
+function walkNonExcludedDirs(root: string): string[] {
+  const dirs: string[] = [root];
+  const queue: string[] = [root];
+
+  while (queue.length > 0) {
+    const current = queue.pop()!;
+    let entries: string[];
+    try {
+      entries = readdirSync(current);
+    } catch {
+      continue; // Permission denied or deleted
+    }
+
+    for (const entry of entries) {
+      // Skip excluded directory names before stat
+      if (EXCLUDED_DIRS.has(entry) || (entry.startsWith(".") && entry !== ".")) continue;
+
+      const fullPath = join(current, entry);
+      try {
+        if (statSync(fullPath).isDirectory()) {
+          dirs.push(fullPath);
+          queue.push(fullPath);
+        }
+      } catch {
+        // stat failed — skip
+      }
+    }
+  }
+
+  return dirs;
+}
+
 export function startWatcher(
   directories: string[],
   options: WatcherOptions
@@ -20,34 +61,54 @@ export function startWatcher(
   const watchers: ReturnType<typeof watch>[] = [];
 
   for (const dir of directories) {
-    try {
-      const watcher = watch(dir, { recursive: true }, (event, filename) => {
-        if (!filename) return;
-        // Accept .md files (indexing) and .jsonl only within .beads/ (Dolt backend)
-        const isMd = filename.endsWith(".md");
-        const isBeadsJsonl = filename.endsWith(".jsonl") && filename.includes(".beads/");
-        if (!isMd && !isBeadsJsonl) return;
-        if (shouldExclude(filename)) return;
-
-        const fullPath = `${dir}/${filename}`;
-        const existing = pending.get(fullPath);
-        if (existing) clearTimeout(existing);
-
-        pending.set(fullPath, setTimeout(async () => {
-          pending.delete(fullPath);
-          try {
-            await onChanged(fullPath, event);
-          } catch (err) {
-            onError?.(err instanceof Error ? err : new Error(String(err)));
-          }
-        }, debounceMs));
-      });
-      watcher.on("error", (err) => {
-        onError?.(err instanceof Error ? err : new Error(String(err)));
-      });
-      watchers.push(watcher);
-    } catch (err) {
-      onError?.(err instanceof Error ? err : new Error(`Failed to watch ${dir}: ${err}`));
+    // Walk the tree, skipping excluded dirs — watch each non-excluded dir individually
+    const watchableDirs = walkNonExcludedDirs(dir);
+
+    // Safety: warn and cap if a single collection path produces too many dirs
+    const MAX_WATCH_DIRS = 500;
+    if (watchableDirs.length > MAX_WATCH_DIRS) {
+      console.log(`[watcher] WARNING: ${dir} has ${watchableDirs.length} dirs — capping at ${MAX_WATCH_DIRS} to prevent FD exhaustion. Consider narrowing the collection path.`);
+      watchableDirs.length = MAX_WATCH_DIRS;
+    } else {
+      console.log(`[watcher] ${dir}: watching ${watchableDirs.length} dirs`);
+    }
+
+    for (const watchDir of watchableDirs) {
+      try {
+        // Non-recursive watch — each dir watched individually
+        const watcher = watch(watchDir, (event, filename) => {
+          if (!filename) return;
+          // Accept .md files (indexing) and .jsonl only within .beads/ (Dolt backend)
+          const isMd = filename.endsWith(".md");
+          const isBeadsJsonl = filename.endsWith(".jsonl") && filename.includes(".beads/");
+          if (!isMd && !isBeadsJsonl) return;
+
+          const relativeToDirRoot = relative(dir, join(watchDir, filename));
+          if (shouldExclude(relativeToDirRoot)) return;
+
+          const fullPath = join(watchDir, filename);
+          const existing = pending.get(fullPath);
+          if (existing) clearTimeout(existing);
+
+          pending.set(fullPath, setTimeout(async () => {
+            pending.delete(fullPath);
+            try {
+              await onChanged(fullPath, event);
+            } catch (err) {
+              onError?.(err instanceof Error ? err : new Error(String(err)));
+            }
+          }, debounceMs));
+        });
+        watcher.on("error", (err) => {
+          onError?.(err instanceof Error ? err : new Error(String(err)));
+        });
+        watchers.push(watcher);
+      } catch (err) {
+        // Individual dir watch failure is non-fatal — skip it
+        if (onError) {
+          onError(err instanceof Error ? err : new Error(`Failed to watch ${watchDir}: ${err}`));
+        }
+      }
     }
   }