clawmem 0.10.4 → 0.10.6

package/SKILL.md

@@ -332,7 +332,7 @@ The query string directly feeds BM25 (which probes first and can short-circuit t
 **For keyword recall (BM25 path):**
 - 2-5 precise terms, no filler words
 - Code identifiers work: `handleError async`
-- BM25 tokenizes on whitespace and AND's all terms as prefix matches (`perf` matches "performance")
+- BM25 tokenizes on whitespace and separators (`_ - . /` etc., mirroring the index), then AND's all terms as prefix matches (`perf` matches "performance"; `before_compaction` matches docs containing `before` and `compaction`)
 - No phrase search or negation syntax — all terms are positive prefix matches
 - A strong keyword hit (score >= 0.85 with gap >= 0.15) skips expansion entirely — faster results
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clawmem",
-  "version": "0.10.4",
+  "version": "0.10.6",
   "description": "On-device memory layer for AI agents. Claude Code, OpenClaw, and Hermes. Hooks + MCP server + hybrid RAG search.",
   "type": "module",
   "bin": {

package/src/amem.ts

@@ -588,6 +588,25 @@ export interface MemoryLink {
   reasoning: string;
 }
 
+function isVectorIndexUnavailable(error: unknown): boolean {
+  const message = String((error as any)?.message ?? error).toLowerCase();
+  return message.includes("no such table: vectors_vec") ||
+    message.includes("no such column: hash_seq") ||
+    message.includes("no such column: embedding") ||
+    message.includes("no such column: v1.embedding") ||
+    message.includes("no such column: v2.embedding") ||
+    message.includes("no such function: vec_distance_cosine") ||
+    message.includes("no such module: vec0");
+}
+
+function isSqlSchemaError(error: unknown): boolean {
+  const message = String((error as any)?.message ?? error).toLowerCase();
+  return message.includes("no such column:") ||
+    message.includes("no such table:") ||
+    message.includes("no such function:") ||
+    message.includes("no such module:");
+}
+
 /**
  * Generate typed memory links for a document based on semantic similarity.
  * Finds k-nearest neighbors and uses LLM to determine relationship types.
@@ -617,26 +636,49 @@ export async function generateMemoryLinks(
       return 0;
     }
 
+    // Indexing may run before vector tables/embeddings are ready.
+    // Degrade quietly here instead of surfacing misleading SQL errors.
+    const vecTableExists = !!store.db.prepare(
+      `SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`
+    ).get();
+    if (!vecTableExists) return 0;
+
     // Find k-nearest neighbors using vector similarity
-    const neighbors = store.db.prepare(`
-      SELECT
-        d2.id as target_id,
-        d2.title as target_title,
-        d2.amem_context as target_context,
-        vec_distance_cosine(v1.embedding, v2.embedding) as distance
-      FROM vectors_vec v1, vectors_vec v2
-      JOIN documents d2 ON v2.hash_seq = d2.hash || '_0'
-      WHERE v1.hash_seq = ? || '_0'
-        AND d2.id != ?
-        AND d2.active = 1
-      ORDER BY distance
-      LIMIT ?
-    `).all(sourceDoc.hash, sourceDoc.id, kNeighbors) as {
+    let neighbors: {
       target_id: number;
       target_title: string;
       target_context: string | null;
       distance: number;
-    }[];
+    }[] = [];
+    try {
+      const sourceVectorExists = !!store.db.prepare(
+        `SELECT 1 FROM vectors_vec WHERE hash_seq = ? LIMIT 1`
+      ).get(`${sourceDoc.hash}_0`);
+      if (!sourceVectorExists) return 0;
+
+      neighbors = store.db.prepare(`
+        SELECT
+          d2.id as target_id,
+          d2.title as target_title,
+          d2.amem_context as target_context,
+          vec_distance_cosine(v1.embedding, v2.embedding) as distance
+        FROM vectors_vec v1, vectors_vec v2
+        JOIN documents d2 ON v2.hash_seq = d2.hash || '_0'
+        WHERE v1.hash_seq = ? || '_0'
+          AND d2.id != ?
+          AND d2.active = 1
+        ORDER BY distance
+        LIMIT ?
+      `).all(sourceDoc.hash, sourceDoc.id, kNeighbors) as {
+        target_id: number;
+        target_title: string;
+        target_context: string | null;
+        distance: number;
+      }[];
+    } catch (error) {
+      if (isVectorIndexUnavailable(error)) return 0;
+      throw error;
+    }
 
     if (neighbors.length === 0) {
       console.log(`[amem] No neighbors found for docId ${docId}`);
@@ -737,6 +779,9 @@ Include all ${neighbors.length} neighbors in your response.`;
     console.log(`[amem] Created ${linksCreated} links for docId ${docId}`);
     return linksCreated;
   } catch (err) {
+    if (isSqlSchemaError(err) && !isVectorIndexUnavailable(err)) {
+      throw err;
+    }
     console.log(`[amem] Error generating memory links for docId ${docId}:`, err);
     return 0;
   }

package/src/entity.ts

@@ -11,6 +11,7 @@ import type { Database } from "bun:sqlite";
 import { createHash } from "crypto";
 import type { LLM } from "./llm.ts";
 import { extractJsonFromLLM } from "./amem.ts";
+import { tokenizeForFTS5 } from "./store.ts";
 
 // =============================================================================
 // Types
@@ -286,6 +287,33 @@ Return ONLY the JSON array. /no_think`;
 // Entity Resolution (canonical normalization)
 // =============================================================================
 
+/**
+ * Exact-first entity FTS candidate gathering.
+ *
+ * `entities_fts` is queried with a LIMIT applied BEFORE the downstream
+ * Levenshtein / mention-count ranking. A pure prefix query (`"tok"*`) can match
+ * a large set (e.g. "Go" -> "go"* matches every "Golang*"), filling the LIMIT
+ * pool and starving the exact row out before it can be ranked. So we gather
+ * exact-token matches first, then top up with prefix matches (deduped by
+ * entity_id) only while under the limit — the exact row is therefore always
+ * present for ranking, and multi-char prefix recall (e.g. "clawme"* ->
+ * "clawmem") is preserved as a supplement. `runMatch` runs the caller's own SQL
+ * because vault scoping / ordering / selected columns differ per call site.
+ */
+function gatherEntityFTSCandidates<T extends { entity_id: string }>(
+  tokens: string[],
+  limit: number,
+  runMatch: (matchExpr: string, lim: number) => T[],
+): T[] {
+  if (tokens.length === 0) return [];
+  const exact = runMatch(tokens.map(t => `"${t}"`).join(' OR '), limit);
+  if (exact.length >= limit) return exact;
+  const seen = new Set(exact.map(r => r.entity_id));
+  const prefix = runMatch(tokens.map(t => `"${t}"*`).join(' OR '), limit)
+    .filter(r => !seen.has(r.entity_id));
+  return exact.concat(prefix).slice(0, limit);
+}
+
 /**
  * Resolve an entity name to its canonical form.
  * Uses FTS5 candidate lookup + Levenshtein fuzzy matching.
@@ -316,13 +344,14 @@ export function resolveEntityCanonical(
   // Step 1: FTS5 candidate lookup — type-agnostic, vault-scoped
   let candidates: { entity_id: string; name: string; entity_type: string }[] = [];
   try {
-    candidates = db.prepare(`
-      SELECT f.entity_id, f.name, f.entity_type
-      FROM entities_fts f
-      JOIN entity_nodes e ON e.entity_id = f.entity_id
-      WHERE entities_fts MATCH ? AND e.vault = ?
-      LIMIT 20
-    `).all(normalizedName.split(/\s+/).map(w => `"${w}"`).join(' OR '), vault) as typeof candidates;
+    candidates = gatherEntityFTSCandidates(tokenizeForFTS5(normalizedName), 20, (matchExpr, lim) =>
+      db.prepare(`
+        SELECT f.entity_id, f.name, f.entity_type
+        FROM entities_fts f
+        JOIN entity_nodes e ON e.entity_id = f.entity_id
+        WHERE entities_fts MATCH ? AND e.vault = ?
+        LIMIT ?
+      `).all(matchExpr, vault, lim) as typeof candidates);
   } catch {
     // FTS5 match may fail on special chars — fall back to LIKE on entity_nodes directly
     candidates = db.prepare(`
@@ -853,14 +882,15 @@ export function searchEntities(
   // Try FTS first
   let results: { entity_id: string; name: string; entity_type: string; mention_count: number }[] = [];
   try {
-    results = db.prepare(`
-      SELECT e.entity_id, e.name, e.entity_type, e.mention_count
-      FROM entities_fts f
-      JOIN entity_nodes e ON e.entity_id = f.entity_id
-      WHERE entities_fts MATCH ?
-      ORDER BY e.mention_count DESC
-      LIMIT ?
-    `).all(normalizedQuery.split(/\s+/).map(w => `"${w}"`).join(' OR '), limit) as typeof results;
+    results = gatherEntityFTSCandidates(tokenizeForFTS5(normalizedQuery), limit, (matchExpr, lim) =>
+      db.prepare(`
+        SELECT e.entity_id, e.name, e.entity_type, e.mention_count
+        FROM entities_fts f
+        JOIN entity_nodes e ON e.entity_id = f.entity_id
+        WHERE entities_fts MATCH ?
+        ORDER BY e.mention_count DESC
+        LIMIT ?
+      `).all(matchExpr, lim) as typeof results);
   } catch {
     // Fallback to LIKE
     results = db.prepare(`

package/src/llm.ts

@@ -332,6 +332,8 @@ export class LlamaCpp implements LLM {
   // Resets after cooldown expires — one network hiccup doesn't permanently disable GPU.
   private remoteEmbedDownUntil = 0;
   private remoteLlmDownUntil = 0;
+  private remoteEmbedFallbackNotifiedUntil = 0;
+  private remoteLlmFallbackNotifiedUntil = 0;
   private static readonly REMOTE_COOLDOWN_MS = 60_000; // 60s cooldown on transport failure
 
   constructor(config: LlamaCppConfig = {}) {
@@ -616,13 +618,21 @@ export class LlamaCpp implements LLM {
       if (result) return result;
       // Cloud providers don't fall back — if API key is set, the user chose cloud
       if (this.isCloudEmbedding()) return null;
+      // HTTP/API errors mean the endpoint is reachable; only transport
+      // failures set cooldown and fall through to local fallback.
+      if (!this.isRemoteEmbedDown()) return null;
       // Transport failure already set cooldown in embedRemote — fall through
     }
 
     // Remote is in cooldown or was never configured — try local fallback
     if (this.remoteEmbedUrl && this.isRemoteEmbedDown()) {
       if (process.env.CLAWMEM_NO_LOCAL_MODELS === "true") return null;
-      console.error("[embed] Remote embed in cooldown, using in-process fallback");
+      this.noteRemoteFallback(
+        "embed",
+        this.isLoopbackUrl(this.remoteEmbedUrl)
+          ? "[embed] Local embedding endpoint unavailable; using in-process fallback during cooldown"
+          : "[embed] Remote embed in cooldown, using in-process fallback"
+      );
     }
 
     // In-process fallback via node-llama-cpp (auto-downloads EmbeddingGemma on first use)
@@ -645,13 +655,21 @@ export class LlamaCpp implements LLM {
       if (results.some(r => r !== null)) return results;
       // Cloud providers don't fall back
       if (this.isCloudEmbedding()) return results;
+      // HTTP/API errors mean the endpoint is reachable; only transport
+      // failures set cooldown and fall through to local fallback.
+      if (!this.isRemoteEmbedDown()) return results;
       // Transport failure already set cooldown in embedRemoteBatch — fall through
     }
 
     // Remote is in cooldown or was never configured — try local fallback
     if (this.remoteEmbedUrl && this.isRemoteEmbedDown()) {
       if (process.env.CLAWMEM_NO_LOCAL_MODELS === "true") return texts.map(() => null);
-      console.error("[embed] Remote embed in cooldown, using in-process fallback");
+      this.noteRemoteFallback(
+        "embed",
+        this.isLoopbackUrl(this.remoteEmbedUrl)
+          ? "[embed] Local embedding endpoint unavailable; using in-process fallback during cooldown"
+          : "[embed] Remote embed in cooldown, using in-process fallback"
+      );
     }
 
     // In-process fallback via node-llama-cpp
@@ -717,7 +735,9 @@ export class LlamaCpp implements LLM {
         code === "UND_ERR_CONNECT_TIMEOUT") return true;
     const msg = String((error as any)?.message || "").toLowerCase();
     if (msg.includes("econnrefused") || msg.includes("etimedout") || msg.includes("enotfound") ||
-        msg.includes("ehostunreach") || msg.includes("enetunreach")) return true;
+        msg.includes("ehostunreach") || msg.includes("enetunreach") ||
+        msg.includes("unable to connect") || msg.includes("connectionrefused") ||
+        msg.includes("connection refused")) return true;
     return false;
   }
 
@@ -734,12 +754,53 @@ export class LlamaCpp implements LLM {
     return Date.now() < this.remoteEmbedDownUntil;
   }
 
+  private isLoopbackUrl(url: string | null | undefined): boolean {
+    if (!url) return false;
+    try {
+      const parsed = new URL(url);
+      return parsed.hostname === "localhost" || parsed.hostname === "127.0.0.1" || parsed.hostname === "::1";
+    } catch {
+      const lower = url.toLowerCase();
+      return lower.includes("localhost") || lower.includes("127.0.0.1") || lower.includes("[::1]");
+    }
+  }
+
+  private noteRemoteFallback(kind: "embed" | "llm", message: string): void {
+    const now = Date.now();
+    if (kind === "embed") {
+      if (now < this.remoteEmbedFallbackNotifiedUntil) return;
+      this.remoteEmbedFallbackNotifiedUntil = this.remoteEmbedDownUntil || (now + LlamaCpp.REMOTE_COOLDOWN_MS);
+      if (this.isLoopbackUrl(this.remoteEmbedUrl)) console.warn(message);
+      else console.error(message);
+      return;
+    }
+
+    if (now < this.remoteLlmFallbackNotifiedUntil) return;
+    this.remoteLlmFallbackNotifiedUntil = this.remoteLlmDownUntil || (now + LlamaCpp.REMOTE_COOLDOWN_MS);
+    if (this.isLoopbackUrl(this.remoteLlmUrl)) console.warn(message);
+    else console.error(message);
+  }
+
   private markRemoteLlmDown(): void {
     this.remoteLlmDownUntil = Date.now() + LlamaCpp.REMOTE_COOLDOWN_MS;
+    this.remoteLlmFallbackNotifiedUntil = 0;
+    this.noteRemoteFallback(
+      "llm",
+      this.isLoopbackUrl(this.remoteLlmUrl)
+        ? "[generate] Local LLM endpoint unavailable, cooldown 60s before retry"
+        : "[generate] Remote LLM unreachable, cooldown 60s before retry"
+    );
   }
 
   private markRemoteEmbedDown(): void {
     this.remoteEmbedDownUntil = Date.now() + LlamaCpp.REMOTE_COOLDOWN_MS;
+    this.remoteEmbedFallbackNotifiedUntil = 0;
+    this.noteRemoteFallback(
+      "embed",
+      this.isLoopbackUrl(this.remoteEmbedUrl)
+        ? "[embed] Local embedding endpoint unavailable, cooldown 60s before retry"
+        : "[embed] Remote embed server unreachable, cooldown 60s before retry"
+    );
   }
 
   // ---------- Remote embedding (GPU server or cloud API via /v1/embeddings) ----------
@@ -840,7 +901,6 @@ export class LlamaCpp implements LLM {
         };
       } catch (error) {
         if (this.isTransportError(error)) {
-          console.error("[embed] Remote embed server unreachable, cooldown 60s");
           this.markRemoteEmbedDown();
         } else {
           console.error("[embed] Remote embed error:", error);
@@ -892,7 +952,6 @@ export class LlamaCpp implements LLM {
         return results;
       } catch (error) {
         if (this.isTransportError(error)) {
-          console.error("[embed] Remote batch embed server unreachable, cooldown 60s");
           this.markRemoteEmbedDown();
         } else {
           console.error("[embed] Remote batch embed error:", error);
@@ -920,7 +979,12 @@ export class LlamaCpp implements LLM {
     // Remote is in cooldown or was never configured — try local fallback
     if (this.remoteLlmUrl && this.isRemoteLlmDown()) {
       if (process.env.CLAWMEM_NO_LOCAL_MODELS === "true") return null;
-      console.error("[generate] Remote LLM in cooldown, falling back to in-process generation");
+      this.noteRemoteFallback(
+        "llm",
+        this.isLoopbackUrl(this.remoteLlmUrl)
+          ? "[generate] Local LLM endpoint unavailable; using in-process generation during cooldown"
+          : "[generate] Remote LLM in cooldown, falling back to in-process generation"
+      );
     }
 
     // Local fallback via node-llama-cpp (CPU)
@@ -1000,7 +1064,6 @@ export class LlamaCpp implements LLM {
         return null;
       }
       if (this.isTransportError(error)) {
-        console.error("[generate] Remote LLM server unreachable, cooldown 60s");
         this.markRemoteLlmDown();
       } else {
         console.error("[generate] Remote LLM error:", error);
@@ -1089,7 +1152,12 @@ Output:`;
         if (includeLexical) fallback.unshift({ type: 'lex', text: query });
         return fallback;
       }
-      console.error("[expandQuery] Remote LLM in cooldown, falling back to in-process grammar expansion");
+      this.noteRemoteFallback(
+        "llm",
+        this.isLoopbackUrl(this.remoteLlmUrl)
+          ? "[expandQuery] Local LLM endpoint unavailable; using in-process grammar expansion during cooldown"
+          : "[expandQuery] Remote LLM in cooldown, falling back to in-process grammar expansion"
+      );
     }
 
     const llama = await this.ensureLlama();

package/src/memory.ts

@@ -254,6 +254,26 @@ export type ScoredResult = EnrichedResult & {
 
 export type CoActivationFn = (path: string) => { path: string; count: number }[];
 
+const STABLE_PROFILE_PATTERNS = [
+  /\b(pronouns?|timezone|time\s*zone|location|locale|home\s+base)\b/i,
+  /\b(preferences?|prefers?|identity|profile|bio|about\s+me)\b/i,
+  /\b(who\s+(am\s+i|is\s+\w+)|what\s+(do\s+i|does\s+\w+)\s+(prefer|use|avoid))\b/i,
+];
+
+function hasStableProfileIntent(query: string): boolean {
+  return STABLE_PROFILE_PATTERNS.some(p => p.test(query));
+}
+
+function canonicalMemoryMultiplier(path: string, contentType: string, query: string): number {
+  if (hasRecencyIntent(query) || !hasStableProfileIntent(query)) return 1.0;
+
+  const lower = path.toLowerCase();
+  if (/(^|\/)(core\/memory|memory\/core|profile|identity|soul)\.md$/.test(lower)) return 1.14;
+  if (/(^|\/)users\/[^/]+\.md$/.test(lower)) return 1.14;
+  if (contentType === "preference") return 1.08;
+  return 1.0;
+}
+
 export function applyCompositeScoring(
   results: EnrichedResult[],
   query: string,
@@ -291,6 +311,8 @@ export function applyCompositeScoring(
     const freqBoost = freqSignal > 0 ? Math.min(0.10, Math.log1p(freqSignal) * 0.03) : 0;
     adjusted *= (1 + freqBoost);
 
+    adjusted *= canonicalMemoryMultiplier(r.displayPath, r.contentType, query);
+
     // Pin boost: +0.3 additive, capped at 1.0
     if (r.pinned) {
       adjusted = Math.min(1.0, adjusted + 0.3);

package/src/openclaw/index.ts

@@ -37,8 +37,8 @@
  *   4. REST API service (`clawmem serve`) lifecycle — unchanged.
  *
  * §14.3 critical correctness contract: `agent_end` is fire-and-forget at
- * `attempt.ts:3379-3402`. Precompact-extract MUST run inside
- * `handleBeforePromptBuild` (which IS awaited at `attempt.ts:2610`), gated
+ * `attempt.ts:3870-3892`. Precompact-extract MUST run inside
+ * `handleBeforePromptBuild` (which IS awaited at `attempt.ts:2973`), gated
  * by the proximity heuristic in `compaction-threshold.ts`. See `engine.ts`
  * top-of-file comment for the full rationale.
  */
@@ -161,7 +161,7 @@ const clawmemPlugin = {
     // ----- Plugin Hook: before_prompt_build (AWAITED — load-bearing path) -----
     // Both context-surfacing retrieval injection and pre-emptive precompact
     // extraction live here. handleBeforePromptBuild is async and the OpenClaw
-    // attempt path awaits the result at attempt.ts:2610 before building the
+    // attempt path awaits the result at attempt.ts:2973 before building the
     // effective prompt. precompact-extract therefore runs strictly before
     // the LLM call that could trigger compaction on this turn.
     api.on(
@@ -175,7 +175,7 @@ const clawmemPlugin = {
     // ----- Plugin Hook: agent_end (FIRE-AND-FORGET in core) -----
     // Decision-extractor, handoff-generator, and feedback-loop run here.
     // These writes are eventually-consistent (saveMemory dedupes), so the
-    // fire-and-forget context at attempt.ts:3379-3402 is acceptable.
+    // fire-and-forget context at attempt.ts:3870-3892 is acceptable.
     // OpenClaw v2026.4.26+ also enforces a 30s default void-hook timeout
     // (DEFAULT_VOID_HOOK_TIMEOUT_MS_BY_HOOK in src/plugins/hooks.ts) — a
     // timed-out handler is logged but our underlying postrun work is not

package/src/store.ts

@@ -298,13 +298,20 @@ if (process.platform === "darwin") {
 }
 
 function initializeDatabase(db: Database): void {
+  // Set busy_timeout FIRST so subsequent PRAGMAs (journal_mode in particular,
+  // which acquires a write lock when switching or initializing WAL state) wait
+  // instead of returning SQLITE_BUSY when concurrent Stop hooks
+  // (decision-extractor, handoff-generator, feedback-loop) — and the parallel
+  // before_reset hook fan-out in src/openclaw/engine.ts — open the DB
+  // simultaneously. busy_timeout is a connection-level setting that only
+  // governs *subsequent* statements (default busy handler is NULL → SQLITE_BUSY
+  // returns immediately), so it must precede the contending PRAGMAs. 15s is
+  // well within the 30s Stop hook timeout. createStore() resets to operational
+  // value (5000ms or opts.busyTimeout) after DDL completes. Issue #13.
+  db.exec("PRAGMA busy_timeout = 15000");
   sqliteVec.load(db);
   db.exec("PRAGMA journal_mode = WAL");
   db.exec("PRAGMA foreign_keys = ON");
-  // Set generous busy_timeout during DDL — concurrent Stop hooks (decision-extractor,
-  // handoff-generator, feedback-loop) all run initializeDatabase simultaneously.
-  // 15s is well within the 30s Stop hook timeout. Reset to normal after DDL completes.
-  db.exec("PRAGMA busy_timeout = 15000");
 
   // Drop legacy tables that are now managed in YAML
   db.exec(`DROP TABLE IF EXISTS path_contexts`);
@@ -1123,13 +1130,20 @@ export function createStore(dbPath?: string, opts?: { readonly?: boolean; busyTi
   if (!opts?.readonly) {
     initializeDatabase(db);
   } else {
-    // Readonly: load sqlite-vec extension and set WAL mode pragma only
+    // Readonly: set busy_timeout FIRST so the journal_mode PRAGMA below
+    // doesn't race when concurrent processes open the DB. PRAGMA
+    // journal_mode=WAL can contend when switching or initializing WAL
+    // state, even on readonly handles. Public-API hardening — no
+    // production caller in this repo currently passes readonly:true,
+    // but the ordering invariant should hold regardless. Issue #13.
+    db.exec(`PRAGMA busy_timeout = ${opts?.busyTimeout ?? 5000}`);
     sqliteVec.load(db);
     db.exec("PRAGMA journal_mode = WAL");
     db.exec("PRAGMA query_only = ON");
   }
-  // Reset busy_timeout to operational value after DDL init (which uses 15s).
-  // Default 5000ms for normal operations — callers can override via opts.
+  // For the writable branch: initializeDatabase() set 15000 during DDL —
+  // reset to operational value here. For readonly: already set inside the
+  // branch above; this assignment is a no-op rewrite to the same value.
   db.exec(`PRAGMA busy_timeout = ${opts?.busyTimeout ?? 5000}`);
 
   return {
@@ -3068,14 +3082,20 @@ export function getTopLevelPathsWithoutContext(db: Database, collectionName: str
 // FTS Search
 // =============================================================================
 
-function sanitizeFTS5Term(term: string): string {
-  return term.replace(/[^\p{L}\p{N}']/gu, '').toLowerCase();
+// Split on any run of non-token chars so query tokenization mirrors the FTS
+// index tokenizer (unicode61) which treats _ - . / ' and all punctuation as
+// token boundaries. Stripping the separators (the old behavior) concatenated
+// word-parts into a token that was never indexed — e.g. "before_compaction"
+// became "beforecompaction" and matched nothing. Shared with entities_fts in
+// entity.ts. Must stay a hoisted `function` declaration: entity.ts imports it
+// across the store<->entity module cycle, which only resolves for hoisted
+// bindings (it is called at runtime, never at module-eval time).
+export function tokenizeForFTS5(query: string): string[] {
+  return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(t => t.length > 0);
 }
 
 function buildFTS5Query(query: string): string | null {
-  const terms = query.split(/\s+/)
-    .map(t => sanitizeFTS5Term(t))
-    .filter(t => t.length > 0);
+  const terms = tokenizeForFTS5(query);
   if (terms.length === 0) return null;
   if (terms.length === 1) return `"${terms[0]}"*`;
   return terms.map(t => `"${t}"*`).join(' AND ');