clawmem 0.16.0 → 0.17.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clawmem",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "On-device memory layer for AI agents. Claude Code, OpenClaw, and Hermes. Hooks + MCP server + hybrid RAG search.",
   "type": "module",
   "bin": {

package/src/clawmem.ts

@@ -9,6 +9,8 @@ import { resolve as pathResolve, basename } from "path";
 import {
   createStore,
   prewarmVectors,
+  startPeriodicPrewarm,
+  resolvePrewarmIntervalMs,
   enableProductionMode,
   getDefaultDbPath,
   canonicalDocId,
@@ -1071,6 +1073,15 @@ function printResults(results: Array<{ displayPath: string; title: string; compo
 // Hook dispatch
 // =============================================================================
 
+// B3: the context-surfacing UserPromptSubmit hook runs under a tight budget
+// (8s repo default). Its OWN writes — dedup UPSERT, context_usage, recall
+// events, co-activations — are all best-effort/fail-open, but under writer
+// contention each could otherwise wait up to the store default busy_timeout
+// (5000ms) and blow the budget. Cap this process's busy_timeout so a contended
+// write fails fast (SQLITE_BUSY → skipped by the fail-open guards) instead of
+// stalling. Reads are unaffected (WAL readers never wait on the write lock).
+const CONTEXT_SURFACING_WRITE_BUSY_TIMEOUT_MS = 1500;
+
 async function cmdHook(args: string[]) {
   const hookName = args[0];
   if (!hookName) die("Usage: clawmem hook <name>");
@@ -1082,6 +1093,11 @@ async function cmdHook(args: string[]) {
   try {
     switch (hookName) {
       case "context-surfacing":
+        // Scope the small busy_timeout to THIS process only. Each `clawmem
+        // hook` invocation runs exactly one hook, so the Stop hooks
+        // (decision-extractor / handoff-generator / feedback-loop, 30s budget)
+        // run in separate processes and keep the store default (5000ms).
+        try { s.db.exec(`PRAGMA busy_timeout = ${CONTEXT_SURFACING_WRITE_BUSY_TIMEOUT_MS}`); } catch { /* non-fatal */ }
         output = await contextSurfacing(s, input);
         break;
       case "session-bootstrap":
@@ -1831,6 +1847,7 @@ async function cmdWatch() {
   let stopHeavyLane: (() => Promise<void>) | null = null;
   let watcherHandle: { close: () => void } | null = null;
   let checkpointTimerHandle: Timer | null = null;
+  let prewarmTimerHandle: ReturnType<typeof setInterval> | null = null;
 
   // Graceful shutdown — stop workers, close watchers, then exit. SIGTERM
   // handling is critical for systemd `systemctl --user stop` to shut down
@@ -1839,6 +1856,13 @@ async function cmdWatch() {
   // its own withWorkerLease finally block before we close the store.
   const shutdown = async (signal: string) => {
     console.log(`\n${c.dim}[watch] Received ${signal}, shutting down...${c.reset}`);
+    // Clear the periodic prewarm FIRST — before the awaited worker drains below. The timer is
+    // unref'd but still fires while the loop is alive; a tick landing mid-drain would run the
+    // synchronous ~1.5 GB scan and delay shutdown. Clearing it here is the only guard against that.
+    if (prewarmTimerHandle) {
+      clearInterval(prewarmTimerHandle);
+      prewarmTimerHandle = null;
+    }
     if (stopHeavyLane) {
       await stopHeavyLane();
       stopHeavyLane = null;
@@ -1901,6 +1925,21 @@ async function cmdWatch() {
     } catch { /* best-effort: unexpected SQL error */ }
   }, 0);
 
+  // B5 Option C: keep the vector payload warm against OS page-cache eviction BETWEEN hook calls.
+  // The one-shot prewarm above warms once; on a long-running host under memory pressure the kernel
+  // can evict the payload and let a cold synchronous MATCH creep back into the context-surfacing
+  // hook path. Re-touching the pages on an interval biases the kernel LRU toward keeping them
+  // resident (a PROBABILITY reduction, not a hard cap — the hard cap is the deferred BACKLOG
+  // Source 46 daemon). Cleared FIRST in shutdown(); the handle is unref'd so it never keeps the
+  // process alive by itself. resolvePrewarmIntervalMs enforces a strict parse + 60s floor so a
+  // stray tiny value (e.g. "1", or "1e3" which parseInt would read as 1) cannot schedule a
+  // near-continuous scan loop. Set CLAWMEM_PREWARM_INTERVAL_MS=0 to disable. Default 10 min.
+  const prewarmIntervalMs = resolvePrewarmIntervalMs(Bun.env.CLAWMEM_PREWARM_INTERVAL_MS);
+  prewarmTimerHandle = startPeriodicPrewarm(s.db, prewarmIntervalMs);
+  if (prewarmTimerHandle) {
+    console.log(`${c.dim}[watch] periodic vector prewarm every ${Math.round(prewarmIntervalMs / 1000)}s${c.reset}`);
+  }
+
   watcherHandle = startWatcher(dirs, {
     debounceMs: 2000,
     onChanged: async (fullPath, event) => {

package/src/hooks/context-surfacing.ts

@@ -149,6 +149,21 @@ export async function contextSurfacing(
   const tokenBudget = profile.tokenBudget;
   const startTime = Date.now();
 
+  // High-fix (B3): the hook's writes to the MAIN store are bounded by the
+  // busy_timeout cmdHook set for this process (1500ms for context-surfacing).
+  // But skill-vault stores are opened separately via resolveStore(), which
+  // would otherwise use the 5000ms operational default — so a contended
+  // skill-vault write (the recall mirror below) could still stall the hook up
+  // to 5s. Inherit the main store's current cap and pass it to EVERY
+  // skill-vault open so those opens/writes are bounded identically. (Reads are
+  // WAL-safe regardless; this primarily bounds the mirror write.)
+  let hookBusyTimeout = 5000;
+  try {
+    const bt = (store.db.prepare("PRAGMA busy_timeout").get() as { timeout?: number } | undefined)?.timeout;
+    if (typeof bt === "number" && bt > 0) hookBusyTimeout = bt;
+  } catch { /* keep default */ }
+  const skillStoreOpts = { busyTimeout: hookBusyTimeout };
+
   // §11.4: Resolve session-scoped focus topic. Primary signal is the
   // per-session focus file at ~/.cache/clawmem/sessions/<id>.focus
   // (file > env var precedence via resolveSessionTopic). Env var
@@ -221,7 +236,7 @@ export async function contextSurfacing(
   // Dual-query: also search skill vault if configured (secondary source)
   if (getVaultPath("skill")) {
     try {
-      const skillStore = resolveStore("skill");
+      const skillStore = resolveStore("skill", skillStoreOpts);
       const skillResults = skillStore.searchFTS(retrievalQuery, 5);
       // Tag skill vault results for identification in output
       for (const r of skillResults) {
@@ -346,7 +361,7 @@ export async function contextSurfacing(
     // expects the collection-relative path, not the full virtual path
     const parsed = r.filepath.startsWith('clawmem://') ? r.filepath.replace(/^clawmem:\/\/[^/]+\/?/, '') : r.filepath;
     // Use the correct store for skill-vault results
-    const targetStore = (r as any)._fromVault === "skill" ? (() => { try { return resolveStore("skill"); } catch { return store; } })() : store;
+    const targetStore = (r as any)._fromVault === "skill" ? (() => { try { return resolveStore("skill", skillStoreOpts); } catch { return store; } })() : store;
     const doc = targetStore.findActiveDocument(r.collectionName, parsed);
     if (!doc) return true;
     if (doc.snoozed_until && new Date(doc.snoozed_until) > now) return false;
@@ -374,7 +389,7 @@ export async function contextSurfacing(
   let enriched = enrichResults(store, generalResults, prompt);
   if (skillResults.length > 0) {
     try {
-      const skillStore = resolveStore("skill");
+      const skillStore = resolveStore("skill", skillStoreOpts);
       enriched = [...enriched, ...enrichResults(skillStore, skillResults, prompt)];
     } catch {
       // Skill store unavailable — enrich with general store as fallback
@@ -500,7 +515,7 @@ export async function contextSurfacing(
           writeRecallEvents(store, input.sessionId, qHash, mappedDocs, validUsageId, turnIndex);
         } else {
           try {
-            const vaultStore = resolveStore(vault);
+            const vaultStore = resolveStore(vault, skillStoreOpts);
             // Mirror context_usage row into named vault for correct FK + attribution
             const vaultPaths = docs.map(r => r.displayPath);
             const vaultUsageId = vaultStore.insertUsage({

package/src/hooks.ts

@@ -213,13 +213,25 @@ export function wasPromptSeenRecently(store: Store, hookName: string, prompt: st
   }
 
   const preview = normalized.slice(0, 120);
-  store.db.prepare(`
-    INSERT INTO hook_dedupe (hook_name, prompt_hash, prompt_preview, last_seen_at)
-    VALUES (?, ?, ?, ?)
-    ON CONFLICT(hook_name, prompt_hash) DO UPDATE SET
-      prompt_preview = excluded.prompt_preview,
-      last_seen_at = excluded.last_seen_at
-  `).run(hookName, hash, preview, nowIso);
+  // Best-effort dedup bookkeeping. Under writer contention this UPSERT can hit
+  // SQLITE_BUSY — especially from the context-surfacing hook, which caps its
+  // busy_timeout low (B3) so its own writes cannot stall the tight
+  // UserPromptSubmit budget. A failed write only means the next identical
+  // prompt won't be suppressed; it is never a reason to throw and abort the
+  // hook. The `recent` verdict comes from the READ above (WAL-safe, does not
+  // wait on the write lock), so same-prompt dedup still works when the row
+  // already exists even if this refresh write is skipped.
+  try {
+    store.db.prepare(`
+      INSERT INTO hook_dedupe (hook_name, prompt_hash, prompt_preview, last_seen_at)
+      VALUES (?, ?, ?, ?)
+      ON CONFLICT(hook_name, prompt_hash) DO UPDATE SET
+        prompt_preview = excluded.prompt_preview,
+        last_seen_at = excluded.last_seen_at
+    `).run(hookName, hash, preview, nowIso);
+  } catch {
+    /* best-effort: contended/failed dedup write must never abort the hook */
+  }
 
   return recent;
 }

package/src/llm.ts

@@ -109,6 +109,14 @@ export type EmbedOptions = {
   model?: string;
   isQuery?: boolean;
   title?: string;
+  /**
+   * Abort signal for the remote embed fetch AND its 429-retry backoff (B4).
+   * The query path passes AbortSignal.timeout(<remaining budget>) so a slow or
+   * rate-limited embed cannot outlive the caller's deadline. Without it, the
+   * hook's Promise.race only ABANDONS the embed promise — the underlying fetch
+   * and retry sleeps keep running; the signal actually CANCELS them.
+   */
+  signal?: AbortSignal;
 };
 
 /**
@@ -697,7 +705,7 @@ export class LlamaCpp implements LLM {
     // Remote server or cloud API — preferred path
     if (this.remoteEmbedUrl && !this.isRemoteEmbedDown()) {
       const extraParams = this.getCloudEmbedParams(!!options.isQuery);
-      const result = await this.embedRemote(text, extraParams);
+      const result = await this.embedRemote(text, extraParams, undefined, options.signal);
       if (result) return result;
       // Cloud providers don't fall back — if API key is set, the user chose cloud
       if (this.isCloudEmbedding()) return null;
@@ -710,6 +718,13 @@ export class LlamaCpp implements LLM {
     // Remote is in cooldown or was never configured — try local fallback
     if (this.remoteEmbedUrl && this.isRemoteEmbedDown()) {
       if (process.env.CLAWMEM_NO_LOCAL_MODELS === "true") return null;
+      // Medium-fix (B4): a deadline-bounded caller (query path, signal set)
+      // cannot afford a local model load/download during a remote cooldown —
+      // embedLocal ignores the abort signal and can run for seconds/minutes.
+      // Skip the local fallback and let the caller degrade (searchVec → [] →
+      // FTS). Pure-local mode (no remoteEmbedUrl) never enters this branch, so
+      // local-only deployments still embed.
+      if (options.signal) return null;
       this.noteRemoteFallback(
         "embed",
         this.isLoopbackUrl(this.remoteEmbedUrl)
@@ -952,22 +967,47 @@ export class LlamaCpp implements LLM {
     return Math.floor(delayMs * (0.75 + Math.random() * 0.5));
   }
 
-  private async embedRemote(text: string, extraParams: Record<string, unknown> = {}, retries = 5): Promise<EmbeddingResult | null> {
+  /**
+   * Sleep for `ms`, resolving early if `signal` aborts. Returns true if the
+   * wait was cut short by an abort (caller should stop retrying), false if it
+   * slept the full duration. Without this, a 429 backoff (up to 30s) would run
+   * to completion even after the caller's deadline elapsed (B4).
+   */
+  private async abortableDelay(ms: number, signal?: AbortSignal): Promise<boolean> {
+    if (signal?.aborted) return true;
+    if (!signal) {
+      await new Promise(r => setTimeout(r, ms));
+      return false;
+    }
+    return await new Promise<boolean>((resolve) => {
+      const onAbort = () => { clearTimeout(timer); resolve(true); };
+      const timer = setTimeout(() => {
+        signal.removeEventListener("abort", onAbort);
+        resolve(false);
+      }, ms);
+      signal.addEventListener("abort", onAbort, { once: true });
+    });
+  }
+
+  private async embedRemote(text: string, extraParams: Record<string, unknown> = {}, retries = 5, signal?: AbortSignal): Promise<EmbeddingResult | null> {
     if (this.isRemoteEmbedDown()) return null;
     const input = this.truncateForEmbed(text);
     for (let attempt = 0; attempt < retries; attempt++) {
+      if (signal?.aborted) return null; // caller deadline already elapsed — do not start another attempt
       try {
         const body: Record<string, unknown> = { input, model: this.remoteEmbedModel, ...extraParams };
         const resp = await fetch(`${this.remoteEmbedUrl}/v1/embeddings`, {
           method: "POST",
           headers: this.getEmbedHeaders(),
           body: JSON.stringify(body),
+          signal,
         });
         if (resp.status === 429) {
           const retryAfter = this.parseRetryAfter(resp);
           const delay = retryAfter ?? Math.min(1000 * 2 ** attempt, 30000);
-          console.error(`Remote embed rate-limited, retry ${attempt + 1}/${retries} in ${this.jitter(delay)}ms`);
-          await new Promise(r => setTimeout(r, this.jitter(delay)));
+          const jittered = this.jitter(delay);
+          console.error(`Remote embed rate-limited, retry ${attempt + 1}/${retries} in ${jittered}ms`);
+          if (await this.abortableDelay(jittered, signal)) return null; // deadline elapsed during backoff
           continue;
         }
         if (!resp.ok) {
@@ -983,6 +1023,13 @@ export class LlamaCpp implements LLM {
           model: data.model || this.remoteEmbedUrl!,
         };
       } catch (error) {
+        // An abort/timeout is an intentional caller-driven cancellation (the
+        // query-path deadline), NOT a transport failure — do not trip the 60s
+        // remote-down cooldown, which would needlessly force local fallback.
+        const name = (error as { name?: string })?.name;
+        if (signal?.aborted || name === "AbortError" || name === "TimeoutError") {
+          return null;
+        }
         if (this.isTransportError(error)) {
           this.markRemoteEmbedDown();
         } else {

package/src/store.ts

@@ -1162,6 +1162,67 @@ export function prewarmVectors(db: Database): boolean {
   return true;
 }
 
+/**
+ * Keep the sqlite-vec payload resident in the OS page cache by re-running the brute-force
+ * prewarm on an interval. The one-shot prewarm at watcher startup warms the cache ONCE; on a
+ * long-running host under memory pressure the kernel can evict the (potentially ~1.5 GB) vector
+ * payload between hook calls, and the next cold SYNCHRONOUS MATCH in the context-surfacing hook
+ * path can then blow the 8-15s hook budget (bun:sqlite exposes no interrupt, so an in-flight
+ * scan cannot be abandoned). Re-touching the pages biases the kernel LRU toward keeping them
+ * resident — a PROBABILITY reduction, NOT a hard cap. The hard cap (moving the blocking scan off
+ * the hook's event loop so its deadline can fire) is the deferred BACKLOG Source 46 daemon.
+ *
+ * Best-effort: never throws; a per-tick failure is swallowed so the timer keeps running. Returns
+ * the interval handle (the caller MUST clear it on shutdown) or null when disabled (intervalMs
+ * <= 0 or non-finite). The handle is unref'd so it never by itself keeps the process alive.
+ * `onPrewarm(ran)` is an optional observability hook fired after each attempt — `ran` is whether
+ * a scan actually executed (i.e. a dimensioned vector table exists).
+ */
+export function startPeriodicPrewarm(
+  db: Database,
+  intervalMs: number,
+  onPrewarm?: (ran: boolean) => void,
+): ReturnType<typeof setInterval> | null {
+  if (!Number.isFinite(intervalMs) || intervalMs <= 0) return null;
+  const timer = setInterval(() => {
+    let ran = false;
+    try { ran = prewarmVectors(db); } catch { /* best-effort: unexpected SQL error */ }
+    if (onPrewarm) { try { onPrewarm(ran); } catch { /* observer must never break the timer */ } }
+  }, intervalMs);
+  (timer as { unref?: () => void }).unref?.();
+  return timer;
+}
+
+/** Floor for the periodic re-prewarm interval. Below this, a large-vault (~1.5 GB) brute-force scan
+ *  runs near-continuously and can saturate the watcher event loop + I/O, so any smaller positive
+ *  request is clamped UP to this value. */
+export const PREWARM_MIN_INTERVAL_MS = 60_000;
+/** Default periodic re-prewarm interval when CLAWMEM_PREWARM_INTERVAL_MS is unset or unparseable. */
+export const PREWARM_DEFAULT_INTERVAL_MS = 600_000;
+
+/**
+ * Resolve the raw CLAWMEM_PREWARM_INTERVAL_MS env value into a SAFE interval for the watcher. This
+ * policy is kept OUT of the permissive `startPeriodicPrewarm` mechanism (so unit tests can still use
+ * tiny intervals) and applied only on the production env path.
+ *   - unset / empty / unparseable → default (600000). Garbage must NOT silently disable the
+ *     mitigation, nor be read as a tiny interval.
+ *   - exactly 0 → 0 (the documented off switch; `startPeriodicPrewarm` then returns null).
+ *   - negative → default (nonsensical; neither an intentional disable nor a fast loop).
+ *   - 0 < n < floor → clamped UP to the 60s floor. Prevents the near-continuous scan loop that e.g.
+ *     "1" or "1e3" would otherwise schedule. NOTE: `Number("1e3") === 1000` whereas
+ *     `parseInt("1e3", 10) === 1`, so `Number()` is used deliberately (parseInt silently truncates
+ *     at the "e").
+ *   - n >= floor → floored to an integer and used as-is.
+ */
+export function resolvePrewarmIntervalMs(raw: string | undefined): number {
+  if (raw === undefined || raw.trim() === "") return PREWARM_DEFAULT_INTERVAL_MS;
+  const n = Number(raw);
+  if (!Number.isFinite(n)) return PREWARM_DEFAULT_INTERVAL_MS;
+  if (n === 0) return 0;
+  if (n < 0) return PREWARM_DEFAULT_INTERVAL_MS;
+  return Math.max(PREWARM_MIN_INTERVAL_MS, Math.floor(n));
+}
+
 /**
  * The DISTINCT non-empty embedding models stored in the vault (empty array if no
  * embeddings exist). Used to detect model drift BETWEEN runs — a different model at
@@ -3391,7 +3452,7 @@ export async function searchVec(db: Database, query: string, model: string, limi
   const tableExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
   if (!tableExists) return [];
 
-  const embedding = await getEmbedding(query, model, true);
+  const embedding = await getEmbedding(query, model, true, deadlineMs);
   if (!embedding) return [];
 
   // Guard-defect fix: the caller's Promise.race(vectorTimeout) cannot interrupt the SYNCHRONOUS
@@ -3499,11 +3560,22 @@ export async function searchVec(db: Database, query: string, model: string, limi
 // Embeddings
 // =============================================================================
 
-async function getEmbedding(text: string, model: string, isQuery: boolean): Promise<number[] | null> {
+async function getEmbedding(text: string, model: string, isQuery: boolean, deadlineMs?: number): Promise<number[] | null> {
   const llm = getDefaultLlamaCpp();
   // Format text using the appropriate prompt template
   const formattedText = isQuery ? formatQueryForEmbedding(text) : formatDocForEmbedding(text);
-  const result = await llm.embed(formattedText, { model, isQuery });
+  // B4: bound the remote embed fetch + its 429 backoff to the caller's wall-clock
+  // deadline. Under the context-surfacing hook's Promise.race the abandoned embed
+  // promise otherwise keeps its fetch + retry sleeps running; AbortSignal.timeout
+  // actually cancels them, so a slow/rate-limited embed can no longer outlive the
+  // hook budget.
+  let signal: AbortSignal | undefined;
+  if (deadlineMs !== undefined) {
+    const remaining = deadlineMs - Date.now();
+    if (remaining <= 0) return null; // deadline already elapsed — skip the embed entirely
+    signal = AbortSignal.timeout(remaining);
+  }
+  const result = await llm.embed(formattedText, { model, isQuery, signal });
   return result?.embedding || null;
 }