@wipcomputer/memory-crystal 0.7.34-alpha.2 → 0.7.34-alpha.3

package/dist/migrate.js

@@ -1,25 +1,1721 @@
 #!/usr/bin/env node
-import {
-  Crystal,
-  resolveConfig
-} from "./chunk-2GBYLMEF.js";
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
 
-// src/migrate.ts
-import Database from "better-sqlite3";
-import { existsSync } from "fs";
+// src/llm.ts
+import { existsSync, readFileSync } from "fs";
 import { join } from "path";
+import { homedir } from "os";
+import { execSync } from "child_process";
+function dbCacheGet(key) {
+  if (!_cacheDb) return null;
+  try {
+    const row = _cacheDb.prepare(
+      "SELECT result FROM llm_cache WHERE cache_key = ? AND created_at > ?"
+    ).get(key, new Date(Date.now() - CACHE_TTL_DAYS * 864e5).toISOString());
+    if (row) {
+      _cacheDb.prepare("UPDATE llm_cache SET hit_count = hit_count + 1, last_hit_at = ? WHERE cache_key = ?").run((/* @__PURE__ */ new Date()).toISOString(), key);
+      return row.result;
+    }
+  } catch {
+  }
+  return null;
+}
+function dbCacheSet(key, type, query, intent, result, provider) {
+  if (!_cacheDb) return;
+  try {
+    _cacheDb.prepare(
+      "INSERT OR REPLACE INTO llm_cache (cache_key, cache_type, query, intent, result, provider, created_at, hit_count, last_hit_at) VALUES (?, ?, ?, ?, ?, ?, ?, 0, NULL)"
+    ).run(key, type, query, intent || null, result, provider, (/* @__PURE__ */ new Date()).toISOString());
+  } catch {
+  }
+}
+function getOpSecret(itemName, fieldLabel) {
+  try {
+    const saTokenPath = join(homedir(), ".openclaw/secrets/op-sa-token");
+    if (!existsSync(saTokenPath)) return void 0;
+    const saToken = readFileSync(saTokenPath, "utf-8").trim();
+    const result = execSync(
+      `OP_SERVICE_ACCOUNT_TOKEN="${saToken}" op item get "${itemName}" --vault "Agent Secrets" --fields "${fieldLabel}" --reveal`,
+      { encoding: "utf-8", timeout: 5e3, stdio: ["pipe", "pipe", "pipe"] }
+    ).trim();
+    return result || void 0;
+  } catch {
+    return void 0;
+  }
+}
+async function detectProvider() {
+  if (detectionDone && detectedProvider) return detectedProvider;
+  detectionDone = true;
+  if (samplingServer) {
+    detectedProvider = { provider: "sampling", baseURL: "", apiKey: "", model: "client-selected" };
+    process.stderr.write("[memory-crystal] LLM provider: MCP Sampling (via client)\n");
+    return detectedProvider;
+  }
+  try {
+    const resp = await fetch("http://localhost:18791/v1/models", { signal: AbortSignal.timeout(1e3) });
+    if (resp.ok) {
+      const data = await resp.json();
+      const model = data?.data?.[0]?.id || "default";
+      detectedProvider = { provider: "mlx", baseURL: "http://localhost:18791/v1", apiKey: "not-needed", model };
+      process.stderr.write(`[memory-crystal] LLM provider: MLX (${model})
+`);
+      return detectedProvider;
+    }
+  } catch {
+  }
+  try {
+    const resp = await fetch("http://localhost:11434/api/tags", { signal: AbortSignal.timeout(1e3) });
+    if (resp.ok) {
+      const data = await resp.json();
+      const models = data?.models || [];
+      const embeddingOnly = ["nomic-embed-text", "mxbai-embed", "all-minilm", "snowflake-arctic-embed"];
+      const chatModel = models.find((m) => !embeddingOnly.some((e) => m.name.startsWith(e)));
+      if (chatModel) {
+        detectedProvider = { provider: "ollama", baseURL: "http://localhost:11434/v1", apiKey: "ollama", model: chatModel.name };
+        process.stderr.write(`[memory-crystal] LLM provider: Ollama (${chatModel.name})
+`);
+        return detectedProvider;
+      }
+    }
+  } catch {
+  }
+  const openaiKey = process.env.OPENAI_API_KEY || getOpSecret("OpenAI API", "api key");
+  if (openaiKey) {
+    detectedProvider = { provider: "openai", baseURL: "https://api.openai.com/v1", apiKey: openaiKey, model: "gpt-4o-mini" };
+    process.stderr.write("[memory-crystal] LLM provider: OpenAI API\n");
+    return detectedProvider;
+  }
+  const anthropicKey = process.env.ANTHROPIC_API_KEY || getOpSecret("Anthropic Auth Token - remote bunkers", "Auth Token");
+  if (anthropicKey && !anthropicKey.startsWith("sk-ant-oat")) {
+    detectedProvider = { provider: "anthropic", baseURL: "https://api.anthropic.com", apiKey: anthropicKey, model: "claude-haiku-4-5-20251001" };
+    process.stderr.write("[memory-crystal] LLM provider: Anthropic API\n");
+    return detectedProvider;
+  }
+  detectedProvider = { provider: "none", baseURL: "", apiKey: "", model: "" };
+  process.stderr.write("[memory-crystal] LLM provider: none (deep search unavailable)\n");
+  return detectedProvider;
+}
+async function chatComplete(config, messages, maxTokens = 300) {
+  if (config.provider === "sampling") {
+    return samplingComplete(messages, maxTokens);
+  }
+  if (config.provider === "anthropic") {
+    return anthropicComplete(config, messages, maxTokens);
+  }
+  const resp = await fetch(`${config.baseURL}/chat/completions`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Authorization": `Bearer ${config.apiKey}`
+    },
+    body: JSON.stringify({
+      model: config.model,
+      messages,
+      max_tokens: maxTokens,
+      temperature: 0.7
+    })
+  });
+  if (!resp.ok) throw new Error(`LLM request failed: ${resp.status}`);
+  const data = await resp.json();
+  return data.choices?.[0]?.message?.content || "";
+}
+async function anthropicComplete(config, messages, maxTokens) {
+  const systemMsg = messages.find((m) => m.role === "system");
+  const userMessages = messages.filter((m) => m.role !== "system");
+  const body = {
+    model: config.model,
+    max_tokens: maxTokens,
+    messages: userMessages
+  };
+  if (systemMsg) body.system = systemMsg.content;
+  const resp = await fetch("https://api.anthropic.com/v1/messages", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "x-api-key": config.apiKey,
+      "anthropic-version": "2023-06-01"
+    },
+    body: JSON.stringify(body)
+  });
+  if (!resp.ok) throw new Error(`Anthropic request failed: ${resp.status}`);
+  const data = await resp.json();
+  return data.content?.[0]?.text || "";
+}
+async function samplingComplete(messages, maxTokens) {
+  if (!samplingServer) throw new Error("MCP sampling server not set");
+  const systemMsg = messages.find((m) => m.role === "system");
+  const userMessages = messages.filter((m) => m.role !== "system");
+  const result = await samplingServer.createMessage({
+    messages: userMessages.map((m) => ({
+      role: m.role,
+      content: { type: "text", text: m.content }
+    })),
+    systemPrompt: systemMsg?.content,
+    maxTokens,
+    modelPreferences: {
+      // Request cheap, fast model (Haiku-class). We don't need Opus for query expansion.
+      costPriority: 0.9,
+      speedPriority: 0.8,
+      intelligencePriority: 0.3,
+      hints: [{ name: "haiku" }]
+    }
+  });
+  if (result?.content?.type === "text") return result.content.text;
+  if (typeof result?.content === "string") return result.content;
+  return "";
+}
+async function expandQuery(query, intent) {
+  const cacheKey = intent ? `expand:${query}||${intent}` : `expand:${query}`;
+  const dbCached = dbCacheGet(cacheKey);
+  if (dbCached) {
+    try {
+      return JSON.parse(dbCached);
+    } catch {
+    }
+  }
+  const cached = expansionCache.get(cacheKey);
+  if (cached) return cached;
+  const config = await detectProvider();
+  if (config.provider === "none") return [];
+  try {
+    const intentContext = intent ? `
+Query intent: ${intent}. Use this to guide your variations toward the intended domain.` : "";
+    const result = await chatComplete(config, [
+      { role: "system", content: EXPAND_PROMPT + intentContext },
+      { role: "user", content: query }
+    ], 300);
+    const lines = result.trim().split("\n");
+    const queryLower = query.toLowerCase();
+    const queryTerms = queryLower.replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean);
+    const hasQueryTerm = (text) => {
+      const lower = text.toLowerCase();
+      if (queryTerms.length === 0) return true;
+      return queryTerms.some((term) => lower.includes(term));
+    };
+    const variations = lines.map((line) => {
+      const colonIdx = line.indexOf(":");
+      if (colonIdx === -1) return null;
+      const type = line.slice(0, colonIdx).trim();
+      if (type !== "lex" && type !== "vec" && type !== "hyde") return null;
+      const text = line.slice(colonIdx + 1).trim();
+      if (!text || !hasQueryTerm(text)) return null;
+      return { type, text };
+    }).filter((v) => v !== null);
+    if (variations.length > 0) {
+      expansionCache.set(cacheKey, variations);
+      dbCacheSet(cacheKey, "expansion", query, intent, JSON.stringify(variations), config.provider);
+      return variations;
+    }
+  } catch (err) {
+    process.stderr.write(`[memory-crystal] Query expansion failed: ${err.message}
+`);
+  }
+  const fallback = [
+    { type: "lex", text: query },
+    { type: "vec", text: query },
+    { type: "hyde", text: `Information about ${query}` }
+  ];
+  return fallback;
+}
+async function rerankResults(query, passages) {
+  const config = await detectProvider();
+  if (config.provider === "none") {
+    return passages.map((_, i) => ({ index: i, score: 1 - i * 0.01 }));
+  }
+  const { createHash: createHash2 } = await import("crypto");
+  const contentHash = createHash2("sha256").update(passages.map((p) => p.slice(0, 200)).sort().join("|")).digest("hex").slice(0, 16);
+  const rerankCacheKey = `rerank:${query}||${contentHash}`;
+  const dbCachedRerank = dbCacheGet(rerankCacheKey);
+  if (dbCachedRerank) {
+    try {
+      return JSON.parse(dbCachedRerank);
+    } catch {
+    }
+  }
+  try {
+    const passageList = passages.map((p, i) => `[${i}] ${p.slice(0, 500)}`).join("\n\n");
+    const result = await chatComplete(config, [
+      { role: "system", content: RERANK_PROMPT },
+      { role: "user", content: `Query: ${query}
+
+Passages:
+${passageList}` }
+    ], 200);
+    const results = [];
+    for (const line of result.trim().split("\n")) {
+      const match = line.match(/^(\d+):\s*([\d.]+)/);
+      if (match) {
+        results.push({ index: parseInt(match[1]), score: parseFloat(match[2]) });
+      }
+    }
+    const scored = new Set(results.map((r) => r.index));
+    for (let i = 0; i < passages.length; i++) {
+      if (!scored.has(i)) results.push({ index: i, score: 0 });
+    }
+    const sorted = results.sort((a, b) => b.score - a.score);
+    dbCacheSet(rerankCacheKey, "rerank", query, void 0, JSON.stringify(sorted), config.provider);
+    return sorted;
+  } catch (err) {
+    process.stderr.write(`[memory-crystal] Reranking failed: ${err.message}
+`);
+    return passages.map((_, i) => ({ index: i, score: 1 - i * 0.01 }));
+  }
+}
+var samplingServer, expansionCache, _cacheDb, CACHE_TTL_DAYS, detectedProvider, detectionDone, EXPAND_PROMPT, RERANK_PROMPT;
+var init_llm = __esm({
+  "src/llm.ts"() {
+    "use strict";
+    samplingServer = null;
+    expansionCache = /* @__PURE__ */ new Map();
+    _cacheDb = null;
+    CACHE_TTL_DAYS = parseInt(process.env.CRYSTAL_CACHE_TTL_DAYS || "7", 10);
+    detectedProvider = null;
+    detectionDone = false;
+    EXPAND_PROMPT = `You are a search query expander. Given a search query, generate exactly 3 variations to improve search recall.
+
+Output exactly 3 lines in this format (no other text):
+lex: <keyword-focused variation for full-text search>
+vec: <semantic variation rephrased for embedding similarity>
+hyde: <hypothetical document snippet that would answer this query>
+
+Rules:
+- Each variation must contain at least one term from the original query
+- Keep variations concise (under 30 words each)
+- lex should use specific keywords and synonyms
+- vec should rephrase the intent naturally
+- hyde should be a short passage as if answering the query`;
+    RERANK_PROMPT = `You are a search result re-ranker. Given a query and a list of text passages, rate each passage's relevance to the query.
+
+Output one line per passage in this exact format:
+<index>: <score>
+
+Where index is the passage number (0-based) and score is a float from 0.0 to 1.0.
+- 1.0 = perfectly relevant, directly answers the query
+- 0.7 = highly relevant, closely related
+- 0.4 = somewhat relevant, tangentially related
+- 0.1 = barely relevant
+- 0.0 = not relevant at all
+
+Rate ALL passages. Output nothing else.`;
+  }
+});
+
+// src/search-pipeline.ts
+var search_pipeline_exports = {};
+__export(search_pipeline_exports, {
+  deepSearch: () => deepSearch
+});
+async function deepSearch(crystal, query, options = {}) {
+  const limit = options.limit || 5;
+  const candidateLimit = options.candidateLimit || DEFAULT_CANDIDATE_LIMIT;
+  const intent = options.intent;
+  const filter = options.filter;
+  const explain = options.explain || false;
+  const provider = await detectProvider();
+  if (provider.provider === "none") {
+    return crystal.search(query, limit, filter);
+  }
+  const db = crystal.sqliteDb;
+  if (!db) return crystal.search(query, limit, filter);
+  const sinceDate = filter?.since ? crystal.parseSince(filter.since) : void 0;
+  const untilDate = filter?.until ? crystal.parseSince(filter.until) : void 0;
+  const internalFilter = { ...filter, sinceDate, untilDate };
+  const initialFts = crystal.searchFTS(query, 20, internalFilter);
+  const topScore = initialFts[0]?.score ?? 0;
+  const secondScore = initialFts[1]?.score ?? 0;
+  const hasStrongSignal = !intent && initialFts.length > 0 && topScore >= STRONG_SIGNAL_MIN_SCORE && topScore - secondScore >= STRONG_SIGNAL_MIN_GAP;
+  const expanded = hasStrongSignal ? [] : await expandQuery(query, intent);
+  const allResultLists = [];
+  if (initialFts.length > 0) allResultLists.push(initialFts);
+  const [queryEmbedding] = await crystal.embed([query]);
+  const originalVec = crystal.searchVec(queryEmbedding, 30, internalFilter);
+  if (originalVec.length > 0) allResultLists.push(originalVec);
+  for (const variation of expanded) {
+    if (variation.type === "lex") {
+      const ftsResults = crystal.searchFTS(variation.text, 20, internalFilter);
+      if (ftsResults.length > 0) allResultLists.push(ftsResults);
+    } else {
+      const [embedding] = await crystal.embed([variation.text]);
+      const vecResults = crystal.searchVec(embedding, 20, internalFilter);
+      if (vecResults.length > 0) allResultLists.push(vecResults);
+    }
+  }
+  const weights = allResultLists.map((_, i) => i < 2 ? 2 : 1);
+  const fused = crystal.reciprocalRankFusion(allResultLists, weights);
+  const candidates = fused.slice(0, candidateLimit);
+  if (candidates.length === 0) return [];
+  const ftsScoreMap = /* @__PURE__ */ new Map();
+  const vecScoreMap = /* @__PURE__ */ new Map();
+  if (explain) {
+    for (const r of initialFts) ftsScoreMap.set(r.text.slice(0, 200), r.score);
+    for (const r of originalVec) vecScoreMap.set(r.text.slice(0, 200), r.score);
+  }
+  const passages = candidates.map((c) => c.text.slice(0, 500));
+  const rerankQuery = intent ? `${intent}: ${query}` : query;
+  const reranked = await rerankResults(rerankQuery, passages);
+  const now = Date.now();
+  const blended = reranked.map((r) => {
+    const candidate = candidates[r.index];
+    if (!candidate) return null;
+    const rrfRank = r.index + 1;
+    let rrfWeight;
+    if (rrfRank <= 3) rrfWeight = 0.75;
+    else if (rrfRank <= 10) rrfWeight = 0.6;
+    else rrfWeight = 0.4;
+    const rrfScore = 1 / rrfRank;
+    const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * r.score;
+    const ageDays = candidate.created_at ? (now - new Date(candidate.created_at).getTime()) / 864e5 : 0;
+    const recency = candidate.created_at ? crystal.recencyWeight(ageDays) : 1;
+    const finalScore = blendedScore * recency;
+    const freshness = candidate.created_at ? crystal.freshnessLabel(ageDays) : void 0;
+    const result = {
+      ...candidate,
+      score: finalScore,
+      freshness
+    };
+    if (explain) {
+      const dedup = candidate.text.slice(0, 200);
+      result.explain = {
+        fts_score: ftsScoreMap.get(dedup),
+        vec_score: vecScoreMap.get(dedup),
+        rrf_rank: rrfRank,
+        rrf_score: rrfScore,
+        rerank_score: r.score,
+        recency_weight: recency,
+        final_score: finalScore
+      };
+    }
+    return result;
+  }).filter((r) => r !== null);
+  const sorted = blended.sort((a, b) => b.score - a.score).slice(0, limit);
+  const topNormScore = sorted[0]?.score || 1;
+  return sorted.map((r) => ({ ...r, score: Math.min(r.score / topNormScore * 0.95, 0.95) }));
+}
+var STRONG_SIGNAL_MIN_SCORE, STRONG_SIGNAL_MIN_GAP, DEFAULT_CANDIDATE_LIMIT;
+var init_search_pipeline = __esm({
+  "src/search-pipeline.ts"() {
+    "use strict";
+    init_llm();
+    STRONG_SIGNAL_MIN_SCORE = 0.85;
+    STRONG_SIGNAL_MIN_GAP = 0.15;
+    DEFAULT_CANDIDATE_LIMIT = 40;
+  }
+});
+
+// src/core.ts
+import * as lancedb from "@lancedb/lancedb";
+import Database from "better-sqlite3";
+import * as sqliteVec from "sqlite-vec";
+import { readFileSync as readFileSync2, existsSync as existsSync2, mkdirSync, readdirSync, statSync } from "fs";
+import { execSync as execSync2 } from "child_process";
+import { join as join2, relative, extname, basename } from "path";
+import { createHash } from "crypto";
+import http from "http";
+import https from "https";
+async function embedOpenAI(texts, apiKey, model) {
+  return new Promise((resolve, reject) => {
+    const body = JSON.stringify({ input: texts, model });
+    const req = https.request({
+      hostname: "api.openai.com",
+      path: "/v1/embeddings",
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${apiKey}`,
+        "Content-Length": Buffer.byteLength(body)
+      },
+      timeout: 3e4
+    }, (res) => {
+      let data = "";
+      res.on("data", (chunk) => data += chunk);
+      res.on("end", () => {
+        if (res.statusCode !== 200) {
+          reject(new Error(`OpenAI API error ${res.statusCode}: ${data.slice(0, 200)}`));
+          return;
+        }
+        const parsed = JSON.parse(data);
+        resolve(parsed.data.map((d) => d.embedding));
+      });
+    });
+    req.on("error", reject);
+    req.on("timeout", () => {
+      req.destroy();
+      reject(new Error("OpenAI timeout"));
+    });
+    req.write(body);
+    req.end();
+  });
+}
+async function embedOllama(texts, host, model) {
+  const results = [];
+  for (const text of texts) {
+    const result = await new Promise((resolve, reject) => {
+      const url = new URL("/api/embeddings", host);
+      const body = JSON.stringify({ model, prompt: text });
+      const req = http.request({
+        hostname: url.hostname,
+        port: url.port,
+        path: url.pathname,
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Content-Length": Buffer.byteLength(body)
+        },
+        timeout: 15e3
+      }, (res) => {
+        let data = "";
+        res.on("data", (chunk) => data += chunk);
+        res.on("end", () => {
+          if (res.statusCode !== 200) {
+            reject(new Error(`Ollama error ${res.statusCode}: ${data.slice(0, 200)}`));
+            return;
+          }
+          resolve(JSON.parse(data).embedding);
+        });
+      });
+      req.on("error", reject);
+      req.on("timeout", () => {
+        req.destroy();
+        reject(new Error("Ollama timeout"));
+      });
+      req.write(body);
+      req.end();
+    });
+    results.push(result);
+  }
+  return results;
+}
+async function embedGoogle(texts, apiKey, model) {
+  return new Promise((resolve, reject) => {
+    const body = JSON.stringify({
+      requests: texts.map((text) => ({ model: `models/${model}`, content: { parts: [{ text }] } }))
+    });
+    const req = https.request({
+      hostname: "generativelanguage.googleapis.com",
+      path: `/v1beta/models/${model}:batchEmbedContents?key=${apiKey}`,
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Content-Length": Buffer.byteLength(body)
+      },
+      timeout: 3e4
+    }, (res) => {
+      let data = "";
+      res.on("data", (chunk) => data += chunk);
+      res.on("end", () => {
+        if (res.statusCode !== 200) {
+          reject(new Error(`Google API error ${res.statusCode}: ${data.slice(0, 200)}`));
+          return;
+        }
+        const parsed = JSON.parse(data);
+        resolve(parsed.embeddings.map((e) => e.values));
+      });
+    });
+    req.on("error", reject);
+    req.on("timeout", () => {
+      req.destroy();
+      reject(new Error("Google timeout"));
+    });
+    req.write(body);
+    req.end();
+  });
+}
+var Crystal = class _Crystal {
+  config;
+  lanceDb = null;
+  sqliteDb = null;
+  chunksTable = null;
+  vecDimensions = null;
+  constructor(config) {
+    this.config = config;
+    if (!existsSync2(config.dataDir)) {
+      mkdirSync(config.dataDir, { recursive: true });
+    }
+  }
+  // ── Initialization ──
+  async init() {
+    const lanceDir = join2(this.config.dataDir, "lance");
+    const sqlitePath = join2(this.config.dataDir, "crystal.db");
+    if (!existsSync2(lanceDir)) mkdirSync(lanceDir, { recursive: true });
+    this.lanceDb = await lancedb.connect(lanceDir);
+    this.sqliteDb = new Database(sqlitePath);
+    this.sqliteDb.pragma("journal_mode = WAL");
+    sqliteVec.load(this.sqliteDb);
+    this.initSqliteTables();
+    this.initChunksTables();
+    await this.initLanceTables();
+  }
+  initSqliteTables() {
+    const db = this.sqliteDb;
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS sources (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        type TEXT NOT NULL,
+        uri TEXT NOT NULL,
+        title TEXT,
+        agent_id TEXT NOT NULL,
+        metadata TEXT DEFAULT '{}',
+        ingested_at TEXT NOT NULL,
+        chunk_count INTEGER DEFAULT 0
+      );
+
+      CREATE TABLE IF NOT EXISTS capture_state (
+        agent_id TEXT NOT NULL,
+        source_id TEXT NOT NULL,
+        last_message_count INTEGER DEFAULT 0,
+        capture_count INTEGER DEFAULT 0,
+        last_capture_at TEXT,
+        PRIMARY KEY (agent_id, source_id)
+      );
+
+      CREATE TABLE IF NOT EXISTS memories (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        text TEXT NOT NULL,
+        category TEXT NOT NULL DEFAULT 'fact',
+        confidence REAL NOT NULL DEFAULT 1.0,
+        source_ids TEXT DEFAULT '[]',
+        status TEXT NOT NULL DEFAULT 'active',
+        created_at TEXT NOT NULL,
+        updated_at TEXT NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS entities (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        name TEXT NOT NULL UNIQUE,
+        type TEXT NOT NULL DEFAULT 'concept',
+        description TEXT,
+        properties TEXT DEFAULT '{}',
+        created_at TEXT NOT NULL,
+        updated_at TEXT NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS relationships (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        source_id INTEGER NOT NULL REFERENCES entities(id),
+        target_id INTEGER NOT NULL REFERENCES entities(id),
+        type TEXT NOT NULL,
+        description TEXT,
+        weight REAL DEFAULT 1.0,
+        valid_from TEXT NOT NULL,
+        valid_until TEXT,
+        created_at TEXT NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_sources_agent ON sources(agent_id);
+      CREATE INDEX IF NOT EXISTS idx_memories_status ON memories(status);
+      CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
+      CREATE INDEX IF NOT EXISTS idx_relationships_source ON relationships(source_id);
+      CREATE INDEX IF NOT EXISTS idx_relationships_target ON relationships(target_id);
+
+      -- LLM cache (persistent expansion + reranking results)
+      CREATE TABLE IF NOT EXISTS llm_cache (
+        cache_key TEXT PRIMARY KEY,
+        cache_type TEXT NOT NULL,
+        query TEXT NOT NULL,
+        intent TEXT,
+        result TEXT NOT NULL,
+        provider TEXT NOT NULL,
+        created_at TEXT NOT NULL,
+        hit_count INTEGER DEFAULT 0,
+        last_hit_at TEXT
+      );
+      CREATE INDEX IF NOT EXISTS idx_llm_cache_type ON llm_cache(cache_type);
+      CREATE INDEX IF NOT EXISTS idx_llm_cache_created ON llm_cache(created_at);
+
+      -- Source file indexing (optional feature)
+      CREATE TABLE IF NOT EXISTS source_collections (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        name TEXT NOT NULL UNIQUE,
+        root_path TEXT NOT NULL,
+        glob_patterns TEXT NOT NULL DEFAULT '["**/*"]',
+        ignore_patterns TEXT NOT NULL DEFAULT '[]',
+        file_count INTEGER DEFAULT 0,
+        chunk_count INTEGER DEFAULT 0,
+        last_sync_at TEXT,
+        created_at TEXT NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS source_files (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        collection_id INTEGER NOT NULL REFERENCES source_collections(id) ON DELETE CASCADE,
+        file_path TEXT NOT NULL,
+        file_hash TEXT NOT NULL,
+        file_size INTEGER NOT NULL,
+        chunk_count INTEGER DEFAULT 0,
+        last_indexed_at TEXT NOT NULL
+      );
+
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_source_files_path ON source_files(collection_id, file_path);
+      CREATE INDEX IF NOT EXISTS idx_source_files_collection ON source_files(collection_id);
+    `);
+  }
+  initChunksTables() {
+    const db = this.sqliteDb;
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        text TEXT NOT NULL,
+        text_hash TEXT NOT NULL,
+        role TEXT,
+        source_type TEXT,
+        source_id TEXT,
+        agent_id TEXT,
+        token_count INTEGER,
+        created_at TEXT NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_chunks_agent ON chunks(agent_id);
+      CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source_type);
+      CREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks(text_hash);
+      CREATE INDEX IF NOT EXISTS idx_chunks_created ON chunks(created_at);
+
+      -- FTS5 full-text search table
+      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+        text,
+        tokenize='porter unicode61'
+      );
+
+      -- Sync trigger: populate FTS on chunk insert
+      CREATE TRIGGER IF NOT EXISTS chunks_fts_insert AFTER INSERT ON chunks
+      BEGIN
+        INSERT INTO chunks_fts(rowid, text) VALUES (NEW.id, NEW.text);
+      END;
+
+      -- Sync trigger: clean up FTS and vec on chunk delete
+      CREATE TRIGGER IF NOT EXISTS chunks_cleanup AFTER DELETE ON chunks
+      BEGIN
+        DELETE FROM chunks_vec WHERE chunk_id = OLD.id;
+        INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES('delete', OLD.id, OLD.text);
+      END;
+    `);
+    const vecTable = db.prepare(
+      `SELECT name FROM sqlite_master WHERE type='table' AND name='chunks_vec'`
+    ).get();
+    if (vecTable) {
+      try {
+        const row = db.prepare("SELECT embedding FROM chunks_vec LIMIT 1").get();
+        if (row?.embedding) {
+          this.vecDimensions = row.embedding.length / 4;
+        }
+      } catch {
+      }
+    }
+  }
+  ensureVecTable(dimensions) {
+    const db = this.sqliteDb;
+    const existing = db.prepare(
+      `SELECT name FROM sqlite_master WHERE type='table' AND name='chunks_vec'`
+    ).get();
+    if (!existing) {
+      db.exec(`
+        CREATE VIRTUAL TABLE chunks_vec USING vec0(
+          chunk_id INTEGER PRIMARY KEY,
+          embedding float[${dimensions}] distance_metric=cosine
+        );
+      `);
+    }
+    this.vecDimensions = dimensions;
+  }
+  async initLanceTables() {
+    const db = this.lanceDb;
+    const tableNames = await db.tableNames();
+    if (tableNames.includes("chunks")) {
+      this.chunksTable = await db.openTable("chunks");
+    }
+  }
+  // ── Embedding ──
+  async embed(texts) {
+    if (texts.length === 0) return [];
+    const cfg = this.config;
+    switch (cfg.embeddingProvider) {
+      case "openai": {
+        if (!cfg.openaiApiKey) throw new Error("OpenAI API key required");
+        const model = cfg.openaiModel || "text-embedding-3-small";
+        const maxCharsPerBatch = 8e5;
+        const results = [];
+        let batch = [];
+        let batchChars = 0;
+        for (const text of texts) {
+          if (batchChars + text.length > maxCharsPerBatch && batch.length > 0) {
+            results.push(...await embedOpenAI(batch, cfg.openaiApiKey, model));
+            batch = [];
+            batchChars = 0;
+          }
+          batch.push(text);
+          batchChars += text.length;
+        }
+        if (batch.length > 0) {
+          results.push(...await embedOpenAI(batch, cfg.openaiApiKey, model));
+        }
+        return results;
+      }
+      case "ollama":
+        return embedOllama(texts, cfg.ollamaHost || "http://localhost:11434", cfg.ollamaModel || "nomic-embed-text");
+      case "google":
+        if (!cfg.googleApiKey) throw new Error("Google API key required");
+        return embedGoogle(texts, cfg.googleApiKey, cfg.googleModel || "text-embedding-004");
+      default:
+        throw new Error(`Unknown embedding provider: ${cfg.embeddingProvider}`);
+    }
+  }
+  // ── Chunking ──
+  chunkText(text, targetTokens = 400, overlapTokens = 80) {
+    const targetChars = targetTokens * 4;
+    const overlapChars = overlapTokens * 4;
+    const chunks = [];
+    let start = 0;
+    while (start < text.length) {
+      let end = Math.min(start + targetChars, text.length);
+      if (end < text.length) {
+        const minBreak = start + Math.floor(targetChars * 0.5);
+        const paraBreak = text.lastIndexOf("\n\n", end);
+        if (paraBreak > minBreak) {
+          end = paraBreak;
+        } else {
+          const sentBreak = text.lastIndexOf(". ", end);
+          if (sentBreak > minBreak) {
+            end = sentBreak + 1;
+          }
+        }
+      }
+      const chunk = text.slice(start, end).trim();
+      if (chunk.length > 0) chunks.push(chunk);
+      if (end >= text.length) break;
+      start = end - overlapChars;
+      if (start <= (chunks.length > 0 ? end - targetChars : 0)) {
+        start = end;
+      }
+    }
+    return chunks;
+  }
+  // ── Ingest ──
+  async ingest(chunks) {
+    if (chunks.length === 0) return 0;
+    const db = this.sqliteDb;
+    const newChunks = chunks.filter((c) => {
+      const hash = createHash("sha256").update(c.text).digest("hex");
+      return !db.prepare("SELECT 1 FROM chunks WHERE text_hash = ?").get(hash);
+    });
+    if (newChunks.length === 0) return 0;
+    const texts = newChunks.map((c) => c.text);
+    const embeddings = await this.embed(texts);
+    if (!this.vecDimensions && embeddings.length > 0) {
+      this.ensureVecTable(embeddings[0].length);
+    }
+    const insertChunk = db.prepare(`
+      INSERT INTO chunks (text, text_hash, role, source_type, source_id, agent_id, token_count, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    const insertVec = db.prepare(`
+      INSERT INTO chunks_vec (chunk_id, embedding) VALUES (?, ?)
+    `);
+    const transaction = db.transaction(() => {
+      for (let i = 0; i < newChunks.length; i++) {
+        const c = newChunks[i];
+        const hash = createHash("sha256").update(c.text).digest("hex");
+        const result = insertChunk.run(
+          c.text,
+          hash,
+          c.role,
+          c.source_type,
+          c.source_id,
+          c.agent_id,
+          c.token_count,
+          c.created_at || (/* @__PURE__ */ new Date()).toISOString()
+        );
+        const chunkId = typeof result.lastInsertRowid === "bigint" ? result.lastInsertRowid : BigInt(result.lastInsertRowid);
+        insertVec.run(chunkId, new Float32Array(embeddings[i]));
+      }
+    });
+    transaction();
+    const records = newChunks.map((chunk, i) => ({
+      text: chunk.text,
+      vector: embeddings[i],
+      role: chunk.role,
+      source_type: chunk.source_type,
+      source_id: chunk.source_id,
+      agent_id: chunk.agent_id,
+      token_count: chunk.token_count,
+      created_at: chunk.created_at || (/* @__PURE__ */ new Date()).toISOString()
+    }));
+    try {
+      if (!this.chunksTable) {
+        this.chunksTable = await this.lanceDb.createTable("chunks", records);
+      } else {
+        await this.chunksTable.add(records);
+      }
+    } catch (err) {
+      console.warn("LanceDB dual-write failed (non-fatal):", err.message);
+    }
+    return newChunks.length;
+  }
+  // ── Delta Sync (export/import pre-embedded chunks) ──
+  /** Export interface for delta sync payloads. */
+  static DELTA_VERSION = 1;
+  /** Export chunks with IDs greater than sinceId. Returns pre-embedded chunks for delta sync.
+   *  Core calls this to build delta payloads for Nodes. */
+  exportChunksSince(sinceId) {
+    const db = this.sqliteDb;
+    const rows = db.prepare(`
+      SELECT c.id, c.text, c.text_hash, c.role, c.source_type, c.source_id,
+             c.agent_id, c.token_count, c.created_at, v.embedding
+      FROM chunks c
+      LEFT JOIN chunks_vec v ON v.chunk_id = c.id
+      WHERE c.id > ?
+      ORDER BY c.id ASC
+    `).all(sinceId);
+    return rows.map((row) => ({
+      id: row.id,
+      text: row.text,
+      text_hash: row.text_hash,
+      role: row.role,
+      source_type: row.source_type,
+      source_id: row.source_id,
+      agent_id: row.agent_id,
+      token_count: row.token_count,
+      created_at: row.created_at,
+      // Convert Float32Array buffer to number[] for JSON serialization
+      embedding: row.embedding ? Array.from(new Float32Array(row.embedding.buffer, row.embedding.byteOffset, row.embedding.byteLength / 4)) : null
+    }));
+  }
+  /** Get the highest chunk ID in the database. Used for watermark tracking. */
+  getMaxChunkId() {
+    const db = this.sqliteDb;
+    const row = db.prepare("SELECT MAX(id) as maxId FROM chunks").get();
+    return row.maxId || 0;
+  }
+  /** Import pre-embedded chunks from Core. Node calls this to apply delta payloads.
+   *  Skips chunks that already exist (by text_hash). Does NOT re-embed. */
+  importChunks(exported) {
+    if (exported.length === 0) return 0;
+    const db = this.sqliteDb;
+    const firstWithEmbed = exported.find((c) => c.embedding && c.embedding.length > 0);
+    if (firstWithEmbed && !this.vecDimensions) {
+      this.ensureVecTable(firstWithEmbed.embedding.length);
+    }
+    const insertChunk = db.prepare(`
+      INSERT INTO chunks (text, text_hash, role, source_type, source_id, agent_id, token_count, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    const insertVec = db.prepare(`
+      INSERT INTO chunks_vec (chunk_id, embedding) VALUES (?, ?)
+    `);
+    const checkHash = db.prepare("SELECT 1 FROM chunks WHERE text_hash = ?");
+    let imported = 0;
+    const transaction = db.transaction(() => {
+      for (const chunk of exported) {
+        if (checkHash.get(chunk.text_hash)) continue;
+        const result = insertChunk.run(
+          chunk.text,
+          chunk.text_hash,
+          chunk.role,
+          chunk.source_type,
+          chunk.source_id,
+          chunk.agent_id,
+          chunk.token_count,
+          chunk.created_at
+        );
+        if (chunk.embedding && chunk.embedding.length > 0) {
+          const chunkId = typeof result.lastInsertRowid === "bigint" ? result.lastInsertRowid : BigInt(result.lastInsertRowid);
+          insertVec.run(chunkId, new Float32Array(chunk.embedding));
+        }
+        imported++;
+      }
+    });
+    transaction();
+    return imported;
+  }
+  // ── Recency helpers ──
+  recencyWeight(ageDays) {
+    return Math.max(0.3, Math.exp(-ageDays * 0.1));
+  }
+  /** Parse relative time strings ("24h", "7d", "30d") or ISO dates into ISO date strings. */
+  parseSince(since) {
+    const match = since.match(/^(\d+)(h|d)$/);
+    if (match) {
+      const [, num, unit] = match;
+      const ms = unit === "h" ? parseInt(num) * 36e5 : parseInt(num) * 864e5;
+      return new Date(Date.now() - ms).toISOString();
+    }
+    const parsed = new Date(since);
+    if (!isNaN(parsed.getTime())) return parsed.toISOString();
+    return void 0;
+  }
+  freshnessLabel(ageDays) {
+    if (ageDays < 3) return "fresh";
+    if (ageDays < 7) return "recent";
+    if (ageDays < 14) return "aging";
+    return "stale";
+  }
+  // ── Search (Hybrid: BM25 + Vector + RRF fusion + Recency) ──
+  async search(query, limit = 5, filter) {
+    const db = this.sqliteDb;
+    const sqliteChunks = db.prepare("SELECT COUNT(*) as count FROM chunks").get()?.count || 0;
+    let lanceChunks = 0;
+    if (this.chunksTable) {
+      try {
+        lanceChunks = await this.chunksTable.countRows();
+      } catch {
+      }
+    }
+    if (sqliteChunks === 0 || lanceChunks > 0 && sqliteChunks < lanceChunks * 0.5) {
+      return this.searchLanceFallback(query, limit, filter);
+    }
+    const sinceDate = filter?.since ? this.parseSince(filter.since) : void 0;
+    const untilDate = filter?.until ? this.parseSince(filter.until) : void 0;
+    const [embedding] = await this.embed([query]);
+    const fetchLimit = Math.max(limit * 5, 50);
+    const vecResults = this.searchVec(embedding, fetchLimit, { ...filter, sinceDate, untilDate });
+    const ftsResults = this.searchFTS(query, fetchLimit, { ...filter, sinceDate, untilDate });
+    const fused = this.reciprocalRankFusion([ftsResults, vecResults], [2, 1]);
+    const now = Date.now();
+    const scored = fused.map((r) => {
+      const ageDays = r.created_at ? (now - new Date(r.created_at).getTime()) / 864e5 : 0;
+      const recency = r.created_at ? this.recencyWeight(ageDays) : 1;
+      const rescaled = r.score * recency;
+      return {
+        ...r,
+        score: rescaled,
+        freshness: r.created_at ? this.freshnessLabel(ageDays) : void 0
+      };
+    });
+    const sorted = scored.sort((a, b) => b.score - a.score).slice(0, limit);
+    const topScore = sorted[0]?.score || 1;
+    return sorted.map((r) => ({ ...r, score: Math.min(r.score / topScore * 0.95, 0.95) }));
+  }
+  /** Deep search: query expansion + LLM re-ranking + position-aware blending.
+   *  Falls back to standard search if no LLM provider is available.
+   *  Supports intent disambiguation, candidateLimit tuning, and explain traces. */
+  async deepSearch(query, limit = 5, filter, options) {
+    const { deepSearch: deepSearchFn } = await Promise.resolve().then(() => (init_search_pipeline(), search_pipeline_exports));
+    return deepSearchFn(this, query, { limit, filter, ...options });
+  }
+  /** Structured search: pass pre-expanded queries to skip LLM expansion.
+   *  Each query is typed (lex, vec, hyde) and searched independently, then fused with RRF. */
+  async structuredSearch(queries, limit = 5, filter) {
+    const db = this.sqliteDb;
+    const sinceDate = filter?.since ? this.parseSince(filter.since) : void 0;
+    const untilDate = filter?.until ? this.parseSince(filter.until) : void 0;
+    const internalFilter = { ...filter, sinceDate, untilDate };
+    const allResultLists = [];
+    for (const q of queries) {
+      if (q.type === "lex") {
+        const fts = this.searchFTS(q.text, Math.max(limit * 5, 50), internalFilter);
+        if (fts.length > 0) allResultLists.push(fts);
+      } else {
+        const [embedding] = await this.embed([q.text]);
+        const vec = this.searchVec(embedding, Math.max(limit * 5, 50), internalFilter);
+        if (vec.length > 0) allResultLists.push(vec);
+      }
+    }
+    const weights = allResultLists.map((_, i) => i === 0 ? 2 : 1);
+    const fused = this.reciprocalRankFusion(allResultLists, weights);
+    const now = Date.now();
+    const scored = fused.map((r) => {
+      const ageDays = r.created_at ? (now - new Date(r.created_at).getTime()) / 864e5 : 0;
+      const recency = r.created_at ? this.recencyWeight(ageDays) : 1;
+      return { ...r, score: r.score * recency, freshness: r.created_at ? this.freshnessLabel(ageDays) : void 0 };
+    });
+    const sorted = scored.sort((a, b) => b.score - a.score).slice(0, limit);
+    const topScore = sorted[0]?.score || 1;
+    return sorted.map((r) => ({ ...r, score: Math.min(r.score / topScore * 0.95, 0.95) }));
+  }
+  /** Vector search via sqlite-vec. Two-step pattern: MATCH first, then JOIN. */
+  searchVec(embedding, limit, filter) {
+    const db = this.sqliteDb;
+    if (!this.vecDimensions) return [];
+    const vecRows = db.prepare(`
+      SELECT chunk_id, distance
+      FROM chunks_vec
+      WHERE embedding MATCH ? AND k = ?
+    `).all(new Float32Array(embedding), limit);
+    if (vecRows.length === 0) return [];
+    const ids = vecRows.map((r) => r.chunk_id);
+    const distMap = new Map(vecRows.map((r) => [r.chunk_id, r.distance]));
+    const placeholders = ids.map(() => "?").join(",");
+    let sql = `SELECT id, text, role, source_type, source_id, agent_id, created_at FROM chunks WHERE id IN (${placeholders})`;
+    const params = [...ids];
+    if (filter?.agent_id) {
+      sql += " AND agent_id = ?";
+      params.push(filter.agent_id);
+    }
+    if (filter?.source_type) {
+      sql += " AND source_type = ?";
+      params.push(filter.source_type);
+    }
+    if (filter?.sinceDate) {
+      sql += " AND created_at >= ?";
+      params.push(filter.sinceDate);
+    }
+    if (filter?.untilDate) {
+      sql += " AND created_at < ?";
+      params.push(filter.untilDate);
+    }
+    const rows = db.prepare(sql).all(...params);
+    return rows.map((row) => ({
+      text: row.text,
+      role: row.role,
+      score: 1 - (distMap.get(row.id) || 1),
+      // cosine similarity from distance
+      source_type: row.source_type,
+      source_id: row.source_id,
+      agent_id: row.agent_id,
+      created_at: row.created_at
+    }));
+  }
+  /** Full-text search via FTS5 with BM25 scoring. */
+  searchFTS(query, limit, filter) {
+    const db = this.sqliteDb;
+    const ftsQuery = this.buildFTS5Query(query);
+    if (!ftsQuery) return [];
+    let sql = `
+      SELECT c.id, c.text, c.role, c.source_type, c.source_id, c.agent_id, c.created_at,
+             bm25(chunks_fts) as bm25_score
+      FROM chunks_fts f
+      JOIN chunks c ON c.id = f.rowid
+      WHERE chunks_fts MATCH ?
+    `;
+    const params = [ftsQuery];
+    if (filter?.agent_id) {
+      sql += " AND c.agent_id = ?";
+      params.push(filter.agent_id);
+    }
+    if (filter?.source_type) {
+      sql += " AND c.source_type = ?";
+      params.push(filter.source_type);
+    }
+    if (filter?.sinceDate) {
+      sql += " AND c.created_at >= ?";
+      params.push(filter.sinceDate);
+    }
+    if (filter?.untilDate) {
+      sql += " AND c.created_at < ?";
+      params.push(filter.untilDate);
+    }
+    sql += " ORDER BY bm25_score LIMIT ?";
+    params.push(limit);
+    const rows = db.prepare(sql).all(...params);
+    return rows.map((row) => ({
+      text: row.text,
+      role: row.role,
+      // BM25 scores are negative (lower = better). Normalize to [0..1).
+      // |x| / (1 + |x|) maps: strong(-10)->0.91, medium(-2)->0.67, weak(-0.5)->0.33
+      score: Math.abs(row.bm25_score) / (1 + Math.abs(row.bm25_score)),
+      source_type: row.source_type,
+      source_id: row.source_id,
+      agent_id: row.agent_id,
+      created_at: row.created_at
+    }));
+  }
+  /** Build a safe FTS5 query from user input. */
+  buildFTS5Query(query) {
+    const terms = query.split(/\s+/).map((t) => t.replace(/[^\p{L}\p{N}']/gu, "").toLowerCase()).filter((t) => t.length > 0);
+    if (terms.length === 0) return null;
+    if (terms.length === 1) return `"${terms[0]}"*`;
+    return terms.map((t) => `"${t}"*`).join(" AND ");
+  }
+  /**
+   * Reciprocal Rank Fusion. Ported from QMD (MIT License, Tobi Lutke, 2024-2026).
+   * Fuses multiple ranked result lists into one using RRF scoring.
+   * Uses text content as dedup key (instead of QMD's file path).
+   */
+  reciprocalRankFusion(resultLists, weights = [], k = 60) {
+    const scores = /* @__PURE__ */ new Map();
+    for (let listIdx = 0; listIdx < resultLists.length; listIdx++) {
+      const list = resultLists[listIdx];
+      if (!list) continue;
+      const weight = weights[listIdx] ?? 1;
+      for (let rank = 0; rank < list.length; rank++) {
+        const result = list[rank];
+        if (!result) continue;
+        const rrfContribution = weight / (k + rank + 1);
+        const dedup = result.text.slice(0, 200);
+        const existing = scores.get(dedup);
+        if (existing) {
+          existing.rrfScore += rrfContribution;
+          existing.topRank = Math.min(existing.topRank, rank);
+        } else {
+          scores.set(dedup, {
+            result,
+            rrfScore: rrfContribution,
+            topRank: rank
+          });
+        }
+      }
+    }
+    for (const entry of scores.values()) {
+      if (entry.topRank === 0) {
+        entry.rrfScore += 0.05;
+      } else if (entry.topRank <= 2) {
+        entry.rrfScore += 0.02;
+      }
+    }
+    return Array.from(scores.values()).sort((a, b) => b.rrfScore - a.rrfScore).map((e) => ({ ...e.result, score: e.rrfScore }));
+  }
+  /** LanceDB fallback for search (used when sqlite-vec tables are empty, pre-migration). */
+  async searchLanceFallback(query, limit, filter) {
+    if (!this.chunksTable) return [];
+    const [embedding] = await this.embed([query]);
+    const fetchLimit = Math.max(limit * 3, 30);
+    let queryBuilder = this.chunksTable.vectorSearch(embedding).distanceType("cosine").limit(fetchLimit);
+    if (filter?.agent_id) {
+      queryBuilder = queryBuilder.where(`agent_id = '${filter.agent_id}'`);
+    }
+    if (filter?.source_type) {
+      queryBuilder = queryBuilder.where(`source_type = '${filter.source_type}'`);
+    }
+    const results = await queryBuilder.toArray();
+    const now = Date.now();
+    return results.map((row) => {
+      const cosine = row._distance != null ? 1 - row._distance : 0;
+      const createdAt = row.created_at || "";
+      const ageDays = createdAt ? (now - new Date(createdAt).getTime()) / 864e5 : 0;
+      const weight = createdAt ? this.recencyWeight(ageDays) : 1;
+      return {
+        text: row.text,
+        role: row.role,
+        score: cosine * weight,
+        source_type: row.source_type,
+        source_id: row.source_id,
+        agent_id: row.agent_id,
+        created_at: createdAt,
+        freshness: createdAt ? this.freshnessLabel(ageDays) : void 0
+      };
+    }).sort((a, b) => b.score - a.score).slice(0, limit);
+  }
+  // ── Remember (explicit fact storage) ──
+  async remember(text, category = "fact") {
+    const db = this.sqliteDb;
+    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const stmt = db.prepare(`
+      INSERT INTO memories (text, category, confidence, source_ids, status, created_at, updated_at)
+      VALUES (?, ?, 1.0, '[]', 'active', ?, ?)
+    `);
+    const result = stmt.run(text, category, now, now);
+    await this.ingest([{
+      text,
+      role: "system",
+      source_type: "manual",
+      source_id: `memory:${result.lastInsertRowid}`,
+      agent_id: "system",
+      token_count: Math.ceil(text.length / 4),
+      created_at: now
+    }]);
+    return result.lastInsertRowid;
+  }
+  // ── Forget (deprecate a memory) ──
+  forget(memoryId) {
+    const db = this.sqliteDb;
+    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const result = db.prepare(`
+      UPDATE memories SET status = 'deprecated', updated_at = ? WHERE id = ? AND status = 'active'
+    `).run(now, memoryId);
+    return result.changes > 0;
+  }
+  // ── Status ──
+  async status() {
+    const db = this.sqliteDb;
+    const sqliteChunks = db.prepare("SELECT COUNT(*) as count FROM chunks").get()?.count || 0;
+    let lanceChunks = 0;
+    if (this.chunksTable) {
+      try {
+        lanceChunks = await this.chunksTable.countRows();
+      } catch {
+      }
+    }
+    const chunks = Math.max(sqliteChunks, lanceChunks);
+    const oldest = db.prepare("SELECT MIN(created_at) as ts FROM chunks").get()?.ts || null;
+    const newest = db.prepare("SELECT MAX(created_at) as ts FROM chunks").get()?.ts || null;
+    const memories = db.prepare("SELECT COUNT(*) as count FROM memories WHERE status = ?").get("active")?.count || 0;
+    const sources = db.prepare("SELECT COUNT(*) as count FROM sources").get()?.count || 0;
+    const chunkAgentRows = db.prepare("SELECT DISTINCT agent_id FROM chunks WHERE agent_id IS NOT NULL").all();
+    const sourceAgentRows = db.prepare("SELECT DISTINCT agent_id FROM sources").all();
+    const captureAgentRows = db.prepare("SELECT DISTINCT agent_id FROM capture_state").all();
+    const agents = [.../* @__PURE__ */ new Set([
+      ...chunkAgentRows.map((r) => r.agent_id),
+      ...sourceAgentRows.map((r) => r.agent_id),
+      ...captureAgentRows.map((r) => r.agent_id)
+    ])];
+    const captureInfo = db.prepare(
+      "SELECT COUNT(*) as count, MAX(last_capture_at) as latest FROM capture_state"
+    ).get();
+    return {
+      chunks,
+      memories,
+      sources,
+      agents,
+      oldestChunk: oldest,
+      newestChunk: newest,
+      embeddingProvider: this.config.embeddingProvider,
+      dataDir: this.config.dataDir,
+      capturedSessions: captureInfo?.count || 0,
+      latestCapture: captureInfo?.latest || null
+    };
+  }
+  // ── Capture State (for incremental ingestion) ──
+  getCaptureState(agentId, sourceId) {
+    const db = this.sqliteDb;
+    const row = db.prepare("SELECT last_message_count, capture_count FROM capture_state WHERE agent_id = ? AND source_id = ?").get(agentId, sourceId);
+    if (!row) return { lastMessageCount: 0, captureCount: 0 };
+    return {
+      lastMessageCount: row.last_message_count,
+      captureCount: row.capture_count
+    };
+  }
+  setCaptureState(agentId, sourceId, messageCount, captureCount) {
+    const db = this.sqliteDb;
+    db.prepare(`
+      INSERT OR REPLACE INTO capture_state (agent_id, source_id, last_message_count, capture_count, last_capture_at)
+      VALUES (?, ?, ?, ?, ?)
+    `).run(agentId, sourceId, messageCount, captureCount, (/* @__PURE__ */ new Date()).toISOString());
+  }
+  // ── Source File Indexing (optional feature) ──
+  //
+  // Add directories as "collections", sync to index/re-index changed files.
+  // All source chunks get source_type='file' so they're searchable alongside
+  // conversations and memories. Nothing here is required... you can use MC
+  // without ever touching sources.
+  // Default patterns for files worth indexing
+  static DEFAULT_INCLUDE = [
+    "**/*.ts",
+    "**/*.js",
+    "**/*.tsx",
+    "**/*.jsx",
+    "**/*.py",
+    "**/*.rs",
+    "**/*.go",
+    "**/*.java",
+    "**/*.md",
+    "**/*.txt",
+    "**/*.json",
+    "**/*.yaml",
+    "**/*.yml",
+    "**/*.toml",
+    "**/*.sh",
+    "**/*.bash",
+    "**/*.zsh",
+    "**/*.css",
+    "**/*.html",
+    "**/*.svg",
+    "**/*.sql",
+    "**/*.graphql",
+    "**/*.c",
+    "**/*.cpp",
+    "**/*.h",
+    "**/*.hpp",
+    "**/*.swift",
+    "**/*.kt",
+    "**/*.rb",
+    "**/*.env.example",
+    "**/*.gitignore",
+    "**/Makefile",
+    "**/Dockerfile",
+    "**/Cargo.toml",
+    "**/package.json",
+    "**/tsconfig.json"
+  ];
+  static DEFAULT_IGNORE = [
+    "**/node_modules/**",
+    "**/.git/**",
+    "**/dist/**",
+    "**/build/**",
+    "**/.next/**",
+    "**/.cache/**",
+    "**/coverage/**",
+    "**/__pycache__/**",
+    "**/target/**",
+    "**/vendor/**",
+    "**/.venv/**",
+    "**/*.lock",
+    "**/package-lock.json",
+    "**/yarn.lock",
+    "**/bun.lockb",
+    "**/*.min.js",
+    "**/*.min.css",
+    "**/*.map",
+    "**/*.png",
+    "**/*.jpg",
+    "**/*.jpeg",
+    "**/*.gif",
+    "**/*.ico",
+    "**/*.webp",
+    "**/*.woff",
+    "**/*.woff2",
+    "**/*.ttf",
+    "**/*.eot",
+    "**/*.mp3",
+    "**/*.mp4",
+    "**/*.wav",
+    "**/*.ogg",
+    "**/*.webm",
+    "**/*.zip",
+    "**/*.tar",
+    "**/*.gz",
+    "**/*.br",
+    "**/*.sqlite",
+    "**/*.db",
+    "**/*.lance/**",
+    "**/*.jsonl",
+    "**/secrets/**",
+    "**/.env"
+  ];
+  /** Add a directory as a source collection for indexing. */
+  async sourcesAdd(rootPath, name, options) {
+    const db = this.sqliteDb;
+    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const includePatterns = JSON.stringify(options?.include || _Crystal.DEFAULT_INCLUDE);
+    const ignorePatterns = JSON.stringify(options?.ignore || _Crystal.DEFAULT_IGNORE);
+    const existing = db.prepare("SELECT * FROM source_collections WHERE name = ?").get(name);
+    if (existing) {
+      throw new Error(`Collection "${name}" already exists. Use sourcesSync() to update it.`);
+    }
+    db.prepare(`
+      INSERT INTO source_collections (name, root_path, glob_patterns, ignore_patterns, created_at)
+      VALUES (?, ?, ?, ?, ?)
+    `).run(name, rootPath, includePatterns, ignorePatterns, now);
+    const row = db.prepare("SELECT * FROM source_collections WHERE name = ?").get(name);
+    return row;
+  }
+  /** Remove a source collection and its file records. Chunks remain in LanceDB. */
+  sourcesRemove(name) {
+    const db = this.sqliteDb;
+    const col = db.prepare("SELECT id FROM source_collections WHERE name = ?").get(name);
+    if (!col) return false;
+    db.prepare("DELETE FROM source_files WHERE collection_id = ?").run(col.id);
+    db.prepare("DELETE FROM source_collections WHERE id = ?").run(col.id);
+    return true;
+  }
+  /** Sync a collection: scan files, detect changes, re-index what changed. */
+  async sourcesSync(name, options) {
+    const db = this.sqliteDb;
+    const startTime = Date.now();
+    const batchSize = options?.batchSize || 20;
+    const col = db.prepare("SELECT * FROM source_collections WHERE name = ?").get(name);
+    if (!col) throw new Error(`Collection "${name}" not found. Add it first with sourcesAdd().`);
+    const includePatterns = JSON.parse(col.glob_patterns);
+    const ignorePatterns = JSON.parse(col.ignore_patterns);
+    const files = this.scanDirectory(col.root_path, includePatterns, ignorePatterns);
+    const existingFiles = /* @__PURE__ */ new Map();
+    const rows = db.prepare("SELECT id, file_path, file_hash FROM source_files WHERE collection_id = ?").all(col.id);
+    for (const row of rows) {
+      existingFiles.set(row.file_path, { id: row.id, file_hash: row.file_hash });
+    }
+    let added = 0;
+    let updated = 0;
+    let removed = 0;
+    let chunksAdded = 0;
+    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const toIndex = [];
+    for (const absPath of files) {
+      const relPath = relative(col.root_path, absPath);
+      let content;
+      try {
+        content = readFileSync2(absPath, "utf-8");
+      } catch {
+        continue;
+      }
+      const stat = statSync(absPath);
+      if (stat.size > 500 * 1024) continue;
+      const hash = createHash("sha256").update(content).digest("hex");
+      const existing = existingFiles.get(relPath);
+      if (existing) {
+        existingFiles.delete(relPath);
+        if (existing.file_hash === hash) continue;
+        toIndex.push({ relPath, absPath, hash, size: stat.size, isUpdate: true });
+      } else {
+        toIndex.push({ relPath, absPath, hash, size: stat.size, isUpdate: false });
+      }
+    }
+    if (options?.dryRun) {
+      const newFiles = toIndex.filter((f) => !f.isUpdate).length;
+      const updatedFiles = toIndex.filter((f) => f.isUpdate).length;
+      return {
+        collection: name,
+        added: newFiles,
+        updated: updatedFiles,
+        removed: existingFiles.size,
+        chunks_added: 0,
+        duration_ms: Date.now() - startTime
+      };
+    }
+    for (let i = 0; i < toIndex.length; i += batchSize) {
+      const batch = toIndex.slice(i, i + batchSize);
+      const allChunks = [];
+      for (const file of batch) {
+        const content = readFileSync2(file.absPath, "utf-8");
+        const ext = extname(file.absPath);
+        const fileName = basename(file.absPath);
+        const header = `File: ${file.relPath}
+
+`;
+        const textChunks = this.chunkText(header + content, 400, 80);
+        const fileChunks = textChunks.map((text) => ({
+          text,
+          role: "system",
+          source_type: "file",
+          source_id: `file:${name}:${file.relPath}`,
+          agent_id: "system",
+          token_count: Math.ceil(text.length / 4),
+          created_at: now
+        }));
+        allChunks.push(...fileChunks);
+        if (file.isUpdate) {
+          db.prepare(`
+            UPDATE source_files SET file_hash = ?, file_size = ?, chunk_count = ?, last_indexed_at = ?
+            WHERE collection_id = ? AND file_path = ?
+          `).run(file.hash, file.size, fileChunks.length, now, col.id, file.relPath);
+          updated++;
+        } else {
+          db.prepare(`
+            INSERT INTO source_files (collection_id, file_path, file_hash, file_size, chunk_count, last_indexed_at)
+            VALUES (?, ?, ?, ?, ?, ?)
+          `).run(col.id, file.relPath, file.hash, file.size, fileChunks.length, now);
+          added++;
+        }
+      }
+      if (allChunks.length > 0) {
+        const ingested = await this.ingest(allChunks);
+        chunksAdded += ingested;
+      }
+    }
+    for (const [relPath, { id }] of existingFiles) {
+      db.prepare("DELETE FROM source_files WHERE id = ?").run(id);
+      removed++;
+    }
+    const fileCount = db.prepare("SELECT COUNT(*) as count FROM source_files WHERE collection_id = ?").get(col.id).count;
+    const chunkCount = db.prepare("SELECT SUM(chunk_count) as total FROM source_files WHERE collection_id = ?").get(col.id).total || 0;
+    db.prepare("UPDATE source_collections SET file_count = ?, chunk_count = ?, last_sync_at = ? WHERE id = ?").run(fileCount, chunkCount, now, col.id);
+    return {
+      collection: name,
+      added,
+      updated,
+      removed,
+      chunks_added: chunksAdded,
+      duration_ms: Date.now() - startTime
+    };
+  }
+  /** Get status of all source collections. */
+  sourcesStatus() {
+    const db = this.sqliteDb;
+    const collections = db.prepare("SELECT name, root_path, file_count, chunk_count, last_sync_at FROM source_collections").all();
+    const totalFiles = collections.reduce((sum, c) => sum + c.file_count, 0);
+    const totalChunks = collections.reduce((sum, c) => sum + c.chunk_count, 0);
+    return {
+      collections: collections.map((c) => ({
+        name: c.name,
+        root_path: c.root_path,
+        file_count: c.file_count,
+        chunk_count: c.chunk_count,
+        last_sync_at: c.last_sync_at
+      })),
+      total_files: totalFiles,
+      total_chunks: totalChunks
+    };
+  }
+  /** Scan a directory recursively, matching include/ignore patterns. */
+  scanDirectory(rootPath, includePatterns, ignorePatterns) {
+    const results = [];
+    const allowedExtensions = /* @__PURE__ */ new Set();
+    const allowedExactNames = /* @__PURE__ */ new Set();
+    for (const pattern of includePatterns) {
+      const extMatch = pattern.match(/\*\*\/\*(\.\w+)$/);
+      if (extMatch) {
+        allowedExtensions.add(extMatch[1]);
+      }
+      const nameMatch = pattern.match(/\*\*\/([^*]+)$/);
+      if (nameMatch && !nameMatch[1].startsWith("*.")) {
+        allowedExactNames.add(nameMatch[1]);
+      }
+    }
+    const ignoreDirs = /* @__PURE__ */ new Set();
+    for (const pattern of ignorePatterns) {
+      const dirMatch = pattern.match(/\*\*\/([^/*]+)\/\*\*$/);
+      if (dirMatch) {
+        ignoreDirs.add(dirMatch[1]);
+      }
+    }
+    const ignoreFiles = /* @__PURE__ */ new Set();
+    for (const pattern of ignorePatterns) {
+      const fileMatch = pattern.match(/\*\*\/\*(\.\w+)$/);
+      if (fileMatch) {
+        ignoreFiles.add(fileMatch[1]);
+      }
+      const exactMatch = pattern.match(/\*\*\/([^*]+)$/);
+      if (exactMatch && !exactMatch[1].includes("/")) {
+        ignoreFiles.add(exactMatch[1]);
+      }
+    }
+    const walk = (dir) => {
+      let entries;
+      try {
+        entries = readdirSync(dir);
+      } catch {
+        return;
+      }
+      for (const entry of entries) {
+        const fullPath = join2(dir, entry);
+        let stat;
+        try {
+          stat = statSync(fullPath);
+        } catch {
+          continue;
+        }
+        if (stat.isDirectory()) {
+          if (ignoreDirs.has(entry)) continue;
+          if (entry.startsWith(".")) continue;
+          walk(fullPath);
+        } else if (stat.isFile()) {
+          const ext = extname(entry);
+          if (ignoreFiles.has(ext)) continue;
+          if (ignoreFiles.has(entry)) continue;
+          if (allowedExtensions.has(ext) || allowedExactNames.has(entry)) {
+            results.push(fullPath);
+          }
+        }
+      }
+    };
+    walk(rootPath);
+    return results;
+  }
+  // ── Orphan Cleanup ──
+  /** Clean orphaned entries in chunks_vec and chunks_fts that no longer have
+   *  corresponding rows in the chunks table. Returns counts of what was found/cleaned. */
+  cleanOrphans(options) {
+    const db = this.sqliteDb;
+    const dryRun = options?.dryRun ?? false;
+    const orphanedVec = db.prepare(
+      "SELECT COUNT(*) as cnt FROM chunks_vec WHERE chunk_id NOT IN (SELECT id FROM chunks)"
+    ).get().cnt;
+    const orphanedFts = db.prepare(
+      "SELECT COUNT(*) as cnt FROM chunks_fts WHERE rowid NOT IN (SELECT id FROM chunks)"
+    ).get().cnt;
+    if (dryRun) {
+      return { orphanedVec, orphanedFts, cleanedVec: 0, cleanedFts: 0, dryRun: true };
+    }
+    let cleanedVec = 0;
+    if (orphanedVec > 0) {
+      const ids = db.prepare(
+        "SELECT chunk_id FROM chunks_vec WHERE chunk_id NOT IN (SELECT id FROM chunks)"
+      ).all();
+      const del = db.prepare("DELETE FROM chunks_vec WHERE chunk_id = ?");
+      const BATCH = 1e3;
+      for (let i = 0; i < ids.length; i += BATCH) {
+        const batch = ids.slice(i, i + BATCH);
+        db.transaction(() => {
+          for (const r of batch) {
+            del.run(r.chunk_id);
+            cleanedVec++;
+          }
+        })();
+      }
+    }
+    let cleanedFts = 0;
+    if (orphanedFts > 0) {
+      db.exec("DELETE FROM chunks_fts");
+      db.exec("INSERT INTO chunks_fts(rowid, text) SELECT id, text FROM chunks");
+      cleanedFts = orphanedFts;
+    }
+    return { orphanedVec, orphanedFts, cleanedVec, cleanedFts, dryRun: false };
+  }
+  // ── Cleanup ──
+  close() {
+    this.sqliteDb?.close();
+  }
+};
+function resolveConfig(overrides) {
+  const HOME = process.env.HOME || "";
+  const ldmMemory = join2(HOME, ".ldm", "memory");
+  let dataDir = overrides?.dataDir || process.env.CRYSTAL_DATA_DIR;
+  if (!dataDir) {
+    if (existsSync2(join2(ldmMemory, "crystal.db"))) {
+      dataDir = ldmMemory;
+    } else {
+      const legacyDir = join2(HOME, ".openclaw", "memory-crystal");
+      if (existsSync2(join2(legacyDir, "crystal.db"))) {
+        dataDir = legacyDir;
+      } else {
+        dataDir = ldmMemory;
+      }
+    }
+  }
+  loadEnvFile(join2(dataDir, ".env"));
+  const openaiApiKey = overrides?.openaiApiKey || process.env.OPENAI_API_KEY || opRead("OpenAI API", "api key");
+  const googleApiKey = overrides?.googleApiKey || process.env.GOOGLE_API_KEY || opRead("Google AI", "api key");
+  const remoteToken = overrides?.remoteToken || process.env.CRYSTAL_REMOTE_TOKEN || opRead("Memory Crystal Remote", "token");
+  return {
+    dataDir,
+    embeddingProvider: overrides?.embeddingProvider || process.env.CRYSTAL_EMBEDDING_PROVIDER || "openai",
+    openaiApiKey,
+    openaiModel: overrides?.openaiModel || process.env.CRYSTAL_OPENAI_MODEL || "text-embedding-3-small",
+    ollamaHost: overrides?.ollamaHost || process.env.CRYSTAL_OLLAMA_HOST || "http://localhost:11434",
+    ollamaModel: overrides?.ollamaModel || process.env.CRYSTAL_OLLAMA_MODEL || "nomic-embed-text",
+    googleApiKey,
+    googleModel: overrides?.googleModel || process.env.CRYSTAL_GOOGLE_MODEL || "text-embedding-004",
+    remoteUrl: overrides?.remoteUrl || process.env.CRYSTAL_REMOTE_URL,
+    remoteToken
+  };
+}
+function loadEnvFile(path) {
+  if (!existsSync2(path)) return;
+  const content = readFileSync2(path, "utf8");
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    const eqIdx = trimmed.indexOf("=");
+    if (eqIdx === -1) continue;
+    const key = trimmed.slice(0, eqIdx).trim();
+    let value = trimmed.slice(eqIdx + 1).trim();
+    if (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'")) {
+      value = value.slice(1, -1);
+    }
+    if (key && !process.env[key]) {
+      process.env[key] = value;
+    }
+  }
+}
+function opRead(item, field) {
+  try {
+    const HOME = process.env.HOME || "";
+    let saTokenPath = join2(HOME, ".ldm", "secrets", "op-sa-token");
+    if (!existsSync2(saTokenPath)) {
+      saTokenPath = join2(HOME, ".openclaw", "secrets", "op-sa-token");
+    }
+    if (!existsSync2(saTokenPath)) return void 0;
+    const saToken = readFileSync2(saTokenPath, "utf8").trim();
+    return execSync2(`op read "op://Agent Secrets/${item}/${field}" 2>/dev/null`, {
+      encoding: "utf8",
+      env: { ...process.env, OP_SERVICE_ACCOUNT_TOKEN: saToken },
+      timeout: 1e4
+    }).trim() || void 0;
+  } catch {
+    return void 0;
+  }
+}
+
+// src/migrate.ts
+import Database2 from "better-sqlite3";
+import { existsSync as existsSync3 } from "fs";
+import { join as join3 } from "path";
 var BATCH_SIZE = 50;
 async function main() {
   const args = process.argv.slice(2);
   const dryRun = args.includes("--dry-run");
   const providerFlag = args.find((_, i) => args[i - 1] === "--provider");
-  const openclawHome = process.env.OPENCLAW_HOME || join(process.env.HOME || "", ".openclaw");
-  const sourcePath = join(openclawHome, "memory", "context-embeddings.sqlite");
-  if (!existsSync(sourcePath)) {
+  const openclawHome = process.env.OPENCLAW_HOME || join3(process.env.HOME || "", ".openclaw");
+  const sourcePath = join3(openclawHome, "memory", "context-embeddings.sqlite");
+  if (!existsSync3(sourcePath)) {
     console.error(`Source not found: ${sourcePath}`);
     process.exit(1);
   }
-  const sourceDb = new Database(sourcePath, { readonly: true });
+  const sourceDb = new Database2(sourcePath, { readonly: true });
   sourceDb.pragma("journal_mode = WAL");
   const total = sourceDb.prepare("SELECT COUNT(*) as count FROM conversation_chunks").get().count;
   console.log(`Found ${total} chunks in context-embeddings.sqlite`);