@wipcomputer/memory-crystal 0.7.30 → 0.7.32

package/dist/chunk-TD3P3K32.js

@@ -1,1199 +0,0 @@
-// src/core.ts
-import * as lancedb from "@lancedb/lancedb";
-import Database from "better-sqlite3";
-import * as sqliteVec from "sqlite-vec";
-import { readFileSync, existsSync, mkdirSync, readdirSync, statSync } from "fs";
-import { execSync } from "child_process";
-import { join, relative, extname, basename } from "path";
-import { createHash } from "crypto";
-import http from "http";
-import https from "https";
-async function embedOpenAI(texts, apiKey, model) {
-  return new Promise((resolve, reject) => {
-    const body = JSON.stringify({ input: texts, model });
-    const req = https.request({
-      hostname: "api.openai.com",
-      path: "/v1/embeddings",
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        "Authorization": `Bearer ${apiKey}`,
-        "Content-Length": Buffer.byteLength(body)
-      },
-      timeout: 3e4
-    }, (res) => {
-      let data = "";
-      res.on("data", (chunk) => data += chunk);
-      res.on("end", () => {
-        if (res.statusCode !== 200) {
-          reject(new Error(`OpenAI API error ${res.statusCode}: ${data.slice(0, 200)}`));
-          return;
-        }
-        const parsed = JSON.parse(data);
-        resolve(parsed.data.map((d) => d.embedding));
-      });
-    });
-    req.on("error", reject);
-    req.on("timeout", () => {
-      req.destroy();
-      reject(new Error("OpenAI timeout"));
-    });
-    req.write(body);
-    req.end();
-  });
-}
-async function embedOllama(texts, host, model) {
-  const results = [];
-  for (const text of texts) {
-    const result = await new Promise((resolve, reject) => {
-      const url = new URL("/api/embeddings", host);
-      const body = JSON.stringify({ model, prompt: text });
-      const req = http.request({
-        hostname: url.hostname,
-        port: url.port,
-        path: url.pathname,
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          "Content-Length": Buffer.byteLength(body)
-        },
-        timeout: 15e3
-      }, (res) => {
-        let data = "";
-        res.on("data", (chunk) => data += chunk);
-        res.on("end", () => {
-          if (res.statusCode !== 200) {
-            reject(new Error(`Ollama error ${res.statusCode}: ${data.slice(0, 200)}`));
-            return;
-          }
-          resolve(JSON.parse(data).embedding);
-        });
-      });
-      req.on("error", reject);
-      req.on("timeout", () => {
-        req.destroy();
-        reject(new Error("Ollama timeout"));
-      });
-      req.write(body);
-      req.end();
-    });
-    results.push(result);
-  }
-  return results;
-}
-async function embedGoogle(texts, apiKey, model) {
-  return new Promise((resolve, reject) => {
-    const body = JSON.stringify({
-      requests: texts.map((text) => ({ model: `models/${model}`, content: { parts: [{ text }] } }))
-    });
-    const req = https.request({
-      hostname: "generativelanguage.googleapis.com",
-      path: `/v1beta/models/${model}:batchEmbedContents?key=${apiKey}`,
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        "Content-Length": Buffer.byteLength(body)
-      },
-      timeout: 3e4
-    }, (res) => {
-      let data = "";
-      res.on("data", (chunk) => data += chunk);
-      res.on("end", () => {
-        if (res.statusCode !== 200) {
-          reject(new Error(`Google API error ${res.statusCode}: ${data.slice(0, 200)}`));
-          return;
-        }
-        const parsed = JSON.parse(data);
-        resolve(parsed.embeddings.map((e) => e.values));
-      });
-    });
-    req.on("error", reject);
-    req.on("timeout", () => {
-      req.destroy();
-      reject(new Error("Google timeout"));
-    });
-    req.write(body);
-    req.end();
-  });
-}
-var Crystal = class _Crystal {
-  config;
-  lanceDb = null;
-  sqliteDb = null;
-  chunksTable = null;
-  vecDimensions = null;
-  constructor(config) {
-    this.config = config;
-    if (!existsSync(config.dataDir)) {
-      mkdirSync(config.dataDir, { recursive: true });
-    }
-  }
-  // ── Initialization ──
-  async init() {
-    const lanceDir = join(this.config.dataDir, "lance");
-    const sqlitePath = join(this.config.dataDir, "crystal.db");
-    if (!existsSync(lanceDir)) mkdirSync(lanceDir, { recursive: true });
-    this.lanceDb = await lancedb.connect(lanceDir);
-    this.sqliteDb = new Database(sqlitePath);
-    this.sqliteDb.pragma("journal_mode = WAL");
-    sqliteVec.load(this.sqliteDb);
-    this.initSqliteTables();
-    this.initChunksTables();
-    await this.initLanceTables();
-  }
-  initSqliteTables() {
-    const db = this.sqliteDb;
-    db.exec(`
-      CREATE TABLE IF NOT EXISTS sources (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        type TEXT NOT NULL,
-        uri TEXT NOT NULL,
-        title TEXT,
-        agent_id TEXT NOT NULL,
-        metadata TEXT DEFAULT '{}',
-        ingested_at TEXT NOT NULL,
-        chunk_count INTEGER DEFAULT 0
-      );
-
-      CREATE TABLE IF NOT EXISTS capture_state (
-        agent_id TEXT NOT NULL,
-        source_id TEXT NOT NULL,
-        last_message_count INTEGER DEFAULT 0,
-        capture_count INTEGER DEFAULT 0,
-        last_capture_at TEXT,
-        PRIMARY KEY (agent_id, source_id)
-      );
-
-      CREATE TABLE IF NOT EXISTS memories (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        text TEXT NOT NULL,
-        category TEXT NOT NULL DEFAULT 'fact',
-        confidence REAL NOT NULL DEFAULT 1.0,
-        source_ids TEXT DEFAULT '[]',
-        status TEXT NOT NULL DEFAULT 'active',
-        created_at TEXT NOT NULL,
-        updated_at TEXT NOT NULL
-      );
-
-      CREATE TABLE IF NOT EXISTS entities (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        name TEXT NOT NULL UNIQUE,
-        type TEXT NOT NULL DEFAULT 'concept',
-        description TEXT,
-        properties TEXT DEFAULT '{}',
-        created_at TEXT NOT NULL,
-        updated_at TEXT NOT NULL
-      );
-
-      CREATE TABLE IF NOT EXISTS relationships (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        source_id INTEGER NOT NULL REFERENCES entities(id),
-        target_id INTEGER NOT NULL REFERENCES entities(id),
-        type TEXT NOT NULL,
-        description TEXT,
-        weight REAL DEFAULT 1.0,
-        valid_from TEXT NOT NULL,
-        valid_until TEXT,
-        created_at TEXT NOT NULL
-      );
-
-      CREATE INDEX IF NOT EXISTS idx_sources_agent ON sources(agent_id);
-      CREATE INDEX IF NOT EXISTS idx_memories_status ON memories(status);
-      CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
-      CREATE INDEX IF NOT EXISTS idx_relationships_source ON relationships(source_id);
-      CREATE INDEX IF NOT EXISTS idx_relationships_target ON relationships(target_id);
-
-      -- Source file indexing (optional feature)
-      CREATE TABLE IF NOT EXISTS source_collections (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        name TEXT NOT NULL UNIQUE,
-        root_path TEXT NOT NULL,
-        glob_patterns TEXT NOT NULL DEFAULT '["**/*"]',
-        ignore_patterns TEXT NOT NULL DEFAULT '[]',
-        file_count INTEGER DEFAULT 0,
-        chunk_count INTEGER DEFAULT 0,
-        last_sync_at TEXT,
-        created_at TEXT NOT NULL
-      );
-
-      CREATE TABLE IF NOT EXISTS source_files (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        collection_id INTEGER NOT NULL REFERENCES source_collections(id) ON DELETE CASCADE,
-        file_path TEXT NOT NULL,
-        file_hash TEXT NOT NULL,
-        file_size INTEGER NOT NULL,
-        chunk_count INTEGER DEFAULT 0,
-        last_indexed_at TEXT NOT NULL
-      );
-
-      CREATE UNIQUE INDEX IF NOT EXISTS idx_source_files_path ON source_files(collection_id, file_path);
-      CREATE INDEX IF NOT EXISTS idx_source_files_collection ON source_files(collection_id);
-    `);
-  }
-  initChunksTables() {
-    const db = this.sqliteDb;
-    db.exec(`
-      CREATE TABLE IF NOT EXISTS chunks (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        text TEXT NOT NULL,
-        text_hash TEXT NOT NULL,
-        role TEXT,
-        source_type TEXT,
-        source_id TEXT,
-        agent_id TEXT,
-        token_count INTEGER,
-        created_at TEXT NOT NULL
-      );
-
-      CREATE INDEX IF NOT EXISTS idx_chunks_agent ON chunks(agent_id);
-      CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source_type);
-      CREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks(text_hash);
-      CREATE INDEX IF NOT EXISTS idx_chunks_created ON chunks(created_at);
-
-      -- FTS5 full-text search table
-      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
-        text,
-        tokenize='porter unicode61'
-      );
-
-      -- Sync trigger: populate FTS on chunk insert
-      CREATE TRIGGER IF NOT EXISTS chunks_fts_insert AFTER INSERT ON chunks
-      BEGIN
-        INSERT INTO chunks_fts(rowid, text) VALUES (NEW.id, NEW.text);
-      END;
-    `);
-    const vecTable = db.prepare(
-      `SELECT name FROM sqlite_master WHERE type='table' AND name='chunks_vec'`
-    ).get();
-    if (vecTable) {
-      try {
-        const row = db.prepare("SELECT embedding FROM chunks_vec LIMIT 1").get();
-        if (row?.embedding) {
-          this.vecDimensions = row.embedding.length / 4;
-        }
-      } catch {
-      }
-    }
-  }
-  ensureVecTable(dimensions) {
-    const db = this.sqliteDb;
-    const existing = db.prepare(
-      `SELECT name FROM sqlite_master WHERE type='table' AND name='chunks_vec'`
-    ).get();
-    if (!existing) {
-      db.exec(`
-        CREATE VIRTUAL TABLE chunks_vec USING vec0(
-          chunk_id INTEGER PRIMARY KEY,
-          embedding float[${dimensions}] distance_metric=cosine
-        );
-      `);
-    }
-    this.vecDimensions = dimensions;
-  }
-  async initLanceTables() {
-    const db = this.lanceDb;
-    const tableNames = await db.tableNames();
-    if (tableNames.includes("chunks")) {
-      this.chunksTable = await db.openTable("chunks");
-    }
-  }
-  // ── Embedding ──
-  async embed(texts) {
-    if (texts.length === 0) return [];
-    const cfg = this.config;
-    switch (cfg.embeddingProvider) {
-      case "openai": {
-        if (!cfg.openaiApiKey) throw new Error("OpenAI API key required");
-        const model = cfg.openaiModel || "text-embedding-3-small";
-        const maxCharsPerBatch = 8e5;
-        const results = [];
-        let batch = [];
-        let batchChars = 0;
-        for (const text of texts) {
-          if (batchChars + text.length > maxCharsPerBatch && batch.length > 0) {
-            results.push(...await embedOpenAI(batch, cfg.openaiApiKey, model));
-            batch = [];
-            batchChars = 0;
-          }
-          batch.push(text);
-          batchChars += text.length;
-        }
-        if (batch.length > 0) {
-          results.push(...await embedOpenAI(batch, cfg.openaiApiKey, model));
-        }
-        return results;
-      }
-      case "ollama":
-        return embedOllama(texts, cfg.ollamaHost || "http://localhost:11434", cfg.ollamaModel || "nomic-embed-text");
-      case "google":
-        if (!cfg.googleApiKey) throw new Error("Google API key required");
-        return embedGoogle(texts, cfg.googleApiKey, cfg.googleModel || "text-embedding-004");
-      default:
-        throw new Error(`Unknown embedding provider: ${cfg.embeddingProvider}`);
-    }
-  }
-  // ── Chunking ──
-  chunkText(text, targetTokens = 400, overlapTokens = 80) {
-    const targetChars = targetTokens * 4;
-    const overlapChars = overlapTokens * 4;
-    const chunks = [];
-    let start = 0;
-    while (start < text.length) {
-      let end = Math.min(start + targetChars, text.length);
-      if (end < text.length) {
-        const minBreak = start + Math.floor(targetChars * 0.5);
-        const paraBreak = text.lastIndexOf("\n\n", end);
-        if (paraBreak > minBreak) {
-          end = paraBreak;
-        } else {
-          const sentBreak = text.lastIndexOf(". ", end);
-          if (sentBreak > minBreak) {
-            end = sentBreak + 1;
-          }
-        }
-      }
-      const chunk = text.slice(start, end).trim();
-      if (chunk.length > 0) chunks.push(chunk);
-      if (end >= text.length) break;
-      start = end - overlapChars;
-      if (start <= (chunks.length > 0 ? end - targetChars : 0)) {
-        start = end;
-      }
-    }
-    return chunks;
-  }
-  // ── Ingest ──
-  async ingest(chunks) {
-    if (chunks.length === 0) return 0;
-    const db = this.sqliteDb;
-    const newChunks = chunks.filter((c) => {
-      const hash = createHash("sha256").update(c.text).digest("hex");
-      return !db.prepare("SELECT 1 FROM chunks WHERE text_hash = ?").get(hash);
-    });
-    if (newChunks.length === 0) return 0;
-    const texts = newChunks.map((c) => c.text);
-    const embeddings = await this.embed(texts);
-    if (!this.vecDimensions && embeddings.length > 0) {
-      this.ensureVecTable(embeddings[0].length);
-    }
-    const insertChunk = db.prepare(`
-      INSERT INTO chunks (text, text_hash, role, source_type, source_id, agent_id, token_count, created_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-    `);
-    const insertVec = db.prepare(`
-      INSERT INTO chunks_vec (chunk_id, embedding) VALUES (?, ?)
-    `);
-    const transaction = db.transaction(() => {
-      for (let i = 0; i < newChunks.length; i++) {
-        const c = newChunks[i];
-        const hash = createHash("sha256").update(c.text).digest("hex");
-        const result = insertChunk.run(
-          c.text,
-          hash,
-          c.role,
-          c.source_type,
-          c.source_id,
-          c.agent_id,
-          c.token_count,
-          c.created_at || (/* @__PURE__ */ new Date()).toISOString()
-        );
-        const chunkId = typeof result.lastInsertRowid === "bigint" ? result.lastInsertRowid : BigInt(result.lastInsertRowid);
-        insertVec.run(chunkId, new Float32Array(embeddings[i]));
-      }
-    });
-    transaction();
-    const records = newChunks.map((chunk, i) => ({
-      text: chunk.text,
-      vector: embeddings[i],
-      role: chunk.role,
-      source_type: chunk.source_type,
-      source_id: chunk.source_id,
-      agent_id: chunk.agent_id,
-      token_count: chunk.token_count,
-      created_at: chunk.created_at || (/* @__PURE__ */ new Date()).toISOString()
-    }));
-    try {
-      if (!this.chunksTable) {
-        this.chunksTable = await this.lanceDb.createTable("chunks", records);
-      } else {
-        await this.chunksTable.add(records);
-      }
-    } catch (err) {
-      console.warn("LanceDB dual-write failed (non-fatal):", err.message);
-    }
-    return newChunks.length;
-  }
-  // ── Recency helpers ──
-  recencyWeight(ageDays) {
-    return Math.max(0.3, Math.exp(-ageDays * 0.1));
-  }
-  /** Parse relative time strings ("24h", "7d", "30d") or ISO dates into ISO date strings. */
-  parseSince(since) {
-    const match = since.match(/^(\d+)(h|d)$/);
-    if (match) {
-      const [, num, unit] = match;
-      const ms = unit === "h" ? parseInt(num) * 36e5 : parseInt(num) * 864e5;
-      return new Date(Date.now() - ms).toISOString();
-    }
-    const parsed = new Date(since);
-    if (!isNaN(parsed.getTime())) return parsed.toISOString();
-    return void 0;
-  }
-  freshnessLabel(ageDays) {
-    if (ageDays < 3) return "fresh";
-    if (ageDays < 7) return "recent";
-    if (ageDays < 14) return "aging";
-    return "stale";
-  }
-  // ── Search (Hybrid: BM25 + Vector + RRF fusion + Recency) ──
-  async search(query, limit = 5, filter) {
-    const db = this.sqliteDb;
-    const sqliteChunks = db.prepare("SELECT COUNT(*) as count FROM chunks").get()?.count || 0;
-    let lanceChunks = 0;
-    if (this.chunksTable) {
-      try {
-        lanceChunks = await this.chunksTable.countRows();
-      } catch {
-      }
-    }
-    if (sqliteChunks === 0 || lanceChunks > 0 && sqliteChunks < lanceChunks * 0.5) {
-      return this.searchLanceFallback(query, limit, filter);
-    }
-    const sinceDate = filter?.since ? this.parseSince(filter.since) : void 0;
-    const [embedding] = await this.embed([query]);
-    const fetchLimit = Math.max(limit * 5, 50);
-    const vecResults = this.searchVec(embedding, fetchLimit, { ...filter, sinceDate });
-    const ftsResults = this.searchFTS(query, fetchLimit, { ...filter, sinceDate });
-    const fused = this.reciprocalRankFusion([ftsResults, vecResults], [2, 1]);
-    const now = Date.now();
-    const scored = fused.map((r) => {
-      const ageDays = r.created_at ? (now - new Date(r.created_at).getTime()) / 864e5 : 0;
-      const recency = r.created_at ? this.recencyWeight(ageDays) : 1;
-      const rescaled = Math.min(r.score * recency * 8, 1);
-      return {
-        ...r,
-        score: rescaled,
-        freshness: r.created_at ? this.freshnessLabel(ageDays) : void 0
-      };
-    });
-    return scored.sort((a, b) => b.score - a.score).slice(0, limit);
-  }
-  /** Vector search via sqlite-vec. Two-step pattern: MATCH first, then JOIN. */
-  searchVec(embedding, limit, filter) {
-    const db = this.sqliteDb;
-    if (!this.vecDimensions) return [];
-    const vecRows = db.prepare(`
-      SELECT chunk_id, distance
-      FROM chunks_vec
-      WHERE embedding MATCH ? AND k = ?
-    `).all(new Float32Array(embedding), limit);
-    if (vecRows.length === 0) return [];
-    const ids = vecRows.map((r) => r.chunk_id);
-    const distMap = new Map(vecRows.map((r) => [r.chunk_id, r.distance]));
-    const placeholders = ids.map(() => "?").join(",");
-    let sql = `SELECT id, text, role, source_type, source_id, agent_id, created_at FROM chunks WHERE id IN (${placeholders})`;
-    const params = [...ids];
-    if (filter?.agent_id) {
-      sql += " AND agent_id = ?";
-      params.push(filter.agent_id);
-    }
-    if (filter?.source_type) {
-      sql += " AND source_type = ?";
-      params.push(filter.source_type);
-    }
-    if (filter?.sinceDate) {
-      sql += " AND created_at >= ?";
-      params.push(filter.sinceDate);
-    }
-    const rows = db.prepare(sql).all(...params);
-    return rows.map((row) => ({
-      text: row.text,
-      role: row.role,
-      score: 1 - (distMap.get(row.id) || 1),
-      // cosine similarity from distance
-      source_type: row.source_type,
-      source_id: row.source_id,
-      agent_id: row.agent_id,
-      created_at: row.created_at
-    }));
-  }
-  /** Full-text search via FTS5 with BM25 scoring. */
-  searchFTS(query, limit, filter) {
-    const db = this.sqliteDb;
-    const ftsQuery = this.buildFTS5Query(query);
-    if (!ftsQuery) return [];
-    let sql = `
-      SELECT c.id, c.text, c.role, c.source_type, c.source_id, c.agent_id, c.created_at,
-             bm25(chunks_fts) as bm25_score
-      FROM chunks_fts f
-      JOIN chunks c ON c.id = f.rowid
-      WHERE chunks_fts MATCH ?
-    `;
-    const params = [ftsQuery];
-    if (filter?.agent_id) {
-      sql += " AND c.agent_id = ?";
-      params.push(filter.agent_id);
-    }
-    if (filter?.source_type) {
-      sql += " AND c.source_type = ?";
-      params.push(filter.source_type);
-    }
-    if (filter?.sinceDate) {
-      sql += " AND c.created_at >= ?";
-      params.push(filter.sinceDate);
-    }
-    sql += " ORDER BY bm25_score LIMIT ?";
-    params.push(limit);
-    const rows = db.prepare(sql).all(...params);
-    return rows.map((row) => ({
-      text: row.text,
-      role: row.role,
-      // BM25 scores are negative (lower = better). Normalize to [0..1).
-      // |x| / (1 + |x|) maps: strong(-10)->0.91, medium(-2)->0.67, weak(-0.5)->0.33
-      score: Math.abs(row.bm25_score) / (1 + Math.abs(row.bm25_score)),
-      source_type: row.source_type,
-      source_id: row.source_id,
-      agent_id: row.agent_id,
-      created_at: row.created_at
-    }));
-  }
-  /** Build a safe FTS5 query from user input. */
-  buildFTS5Query(query) {
-    const terms = query.split(/\s+/).map((t) => t.replace(/[^\p{L}\p{N}']/gu, "").toLowerCase()).filter((t) => t.length > 0);
-    if (terms.length === 0) return null;
-    if (terms.length === 1) return `"${terms[0]}"*`;
-    return terms.map((t) => `"${t}"*`).join(" AND ");
-  }
-  /**
-   * Reciprocal Rank Fusion. Ported from QMD (MIT License, Tobi Lutke, 2024-2026).
-   * Fuses multiple ranked result lists into one using RRF scoring.
-   * Uses text content as dedup key (instead of QMD's file path).
-   */
-  reciprocalRankFusion(resultLists, weights = [], k = 60) {
-    const scores = /* @__PURE__ */ new Map();
-    for (let listIdx = 0; listIdx < resultLists.length; listIdx++) {
-      const list = resultLists[listIdx];
-      if (!list) continue;
-      const weight = weights[listIdx] ?? 1;
-      for (let rank = 0; rank < list.length; rank++) {
-        const result = list[rank];
-        if (!result) continue;
-        const rrfContribution = weight / (k + rank + 1);
-        const dedup = result.text.slice(0, 200);
-        const existing = scores.get(dedup);
-        if (existing) {
-          existing.rrfScore += rrfContribution;
-          existing.topRank = Math.min(existing.topRank, rank);
-        } else {
-          scores.set(dedup, {
-            result,
-            rrfScore: rrfContribution,
-            topRank: rank
-          });
-        }
-      }
-    }
-    for (const entry of scores.values()) {
-      if (entry.topRank === 0) {
-        entry.rrfScore += 0.05;
-      } else if (entry.topRank <= 2) {
-        entry.rrfScore += 0.02;
-      }
-    }
-    return Array.from(scores.values()).sort((a, b) => b.rrfScore - a.rrfScore).map((e) => ({ ...e.result, score: e.rrfScore }));
-  }
-  /** LanceDB fallback for search (used when sqlite-vec tables are empty, pre-migration). */
-  async searchLanceFallback(query, limit, filter) {
-    if (!this.chunksTable) return [];
-    const [embedding] = await this.embed([query]);
-    const fetchLimit = Math.max(limit * 3, 30);
-    let queryBuilder = this.chunksTable.vectorSearch(embedding).distanceType("cosine").limit(fetchLimit);
-    if (filter?.agent_id) {
-      queryBuilder = queryBuilder.where(`agent_id = '${filter.agent_id}'`);
-    }
-    if (filter?.source_type) {
-      queryBuilder = queryBuilder.where(`source_type = '${filter.source_type}'`);
-    }
-    const results = await queryBuilder.toArray();
-    const now = Date.now();
-    return results.map((row) => {
-      const cosine = row._distance != null ? 1 - row._distance : 0;
-      const createdAt = row.created_at || "";
-      const ageDays = createdAt ? (now - new Date(createdAt).getTime()) / 864e5 : 0;
-      const weight = createdAt ? this.recencyWeight(ageDays) : 1;
-      return {
-        text: row.text,
-        role: row.role,
-        score: cosine * weight,
-        source_type: row.source_type,
-        source_id: row.source_id,
-        agent_id: row.agent_id,
-        created_at: createdAt,
-        freshness: createdAt ? this.freshnessLabel(ageDays) : void 0
-      };
-    }).sort((a, b) => b.score - a.score).slice(0, limit);
-  }
-  // ── Remember (explicit fact storage) ──
-  async remember(text, category = "fact") {
-    const db = this.sqliteDb;
-    const now = (/* @__PURE__ */ new Date()).toISOString();
-    const stmt = db.prepare(`
-      INSERT INTO memories (text, category, confidence, source_ids, status, created_at, updated_at)
-      VALUES (?, ?, 1.0, '[]', 'active', ?, ?)
-    `);
-    const result = stmt.run(text, category, now, now);
-    await this.ingest([{
-      text,
-      role: "system",
-      source_type: "manual",
-      source_id: `memory:${result.lastInsertRowid}`,
-      agent_id: "system",
-      token_count: Math.ceil(text.length / 4),
-      created_at: now
-    }]);
-    return result.lastInsertRowid;
-  }
-  // ── Forget (deprecate a memory) ──
-  forget(memoryId) {
-    const db = this.sqliteDb;
-    const now = (/* @__PURE__ */ new Date()).toISOString();
-    const result = db.prepare(`
-      UPDATE memories SET status = 'deprecated', updated_at = ? WHERE id = ? AND status = 'active'
-    `).run(now, memoryId);
-    return result.changes > 0;
-  }
-  // ── Status ──
-  async status() {
-    const db = this.sqliteDb;
-    const sqliteChunks = db.prepare("SELECT COUNT(*) as count FROM chunks").get()?.count || 0;
-    let lanceChunks = 0;
-    if (this.chunksTable) {
-      try {
-        lanceChunks = await this.chunksTable.countRows();
-      } catch {
-      }
-    }
-    const chunks = Math.max(sqliteChunks, lanceChunks);
-    const oldest = db.prepare("SELECT MIN(created_at) as ts FROM chunks").get()?.ts || null;
-    const newest = db.prepare("SELECT MAX(created_at) as ts FROM chunks").get()?.ts || null;
-    const memories = db.prepare("SELECT COUNT(*) as count FROM memories WHERE status = ?").get("active")?.count || 0;
-    const sources = db.prepare("SELECT COUNT(*) as count FROM sources").get()?.count || 0;
-    const chunkAgentRows = db.prepare("SELECT DISTINCT agent_id FROM chunks WHERE agent_id IS NOT NULL").all();
-    const sourceAgentRows = db.prepare("SELECT DISTINCT agent_id FROM sources").all();
-    const captureAgentRows = db.prepare("SELECT DISTINCT agent_id FROM capture_state").all();
-    const agents = [.../* @__PURE__ */ new Set([
-      ...chunkAgentRows.map((r) => r.agent_id),
-      ...sourceAgentRows.map((r) => r.agent_id),
-      ...captureAgentRows.map((r) => r.agent_id)
-    ])];
-    const captureInfo = db.prepare(
-      "SELECT COUNT(*) as count, MAX(last_capture_at) as latest FROM capture_state"
-    ).get();
-    return {
-      chunks,
-      memories,
-      sources,
-      agents,
-      oldestChunk: oldest,
-      newestChunk: newest,
-      embeddingProvider: this.config.embeddingProvider,
-      dataDir: this.config.dataDir,
-      capturedSessions: captureInfo?.count || 0,
-      latestCapture: captureInfo?.latest || null
-    };
-  }
-  // ── Capture State (for incremental ingestion) ──
-  getCaptureState(agentId, sourceId) {
-    const db = this.sqliteDb;
-    const row = db.prepare("SELECT last_message_count, capture_count FROM capture_state WHERE agent_id = ? AND source_id = ?").get(agentId, sourceId);
-    if (!row) return { lastMessageCount: 0, captureCount: 0 };
-    return {
-      lastMessageCount: row.last_message_count,
-      captureCount: row.capture_count
-    };
-  }
-  setCaptureState(agentId, sourceId, messageCount, captureCount) {
-    const db = this.sqliteDb;
-    db.prepare(`
-      INSERT OR REPLACE INTO capture_state (agent_id, source_id, last_message_count, capture_count, last_capture_at)
-      VALUES (?, ?, ?, ?, ?)
-    `).run(agentId, sourceId, messageCount, captureCount, (/* @__PURE__ */ new Date()).toISOString());
-  }
-  // ── Source File Indexing (optional feature) ──
-  //
-  // Add directories as "collections", sync to index/re-index changed files.
-  // All source chunks get source_type='file' so they're searchable alongside
-  // conversations and memories. Nothing here is required... you can use MC
-  // without ever touching sources.
-  // Default patterns for files worth indexing
-  static DEFAULT_INCLUDE = [
-    "**/*.ts",
-    "**/*.js",
-    "**/*.tsx",
-    "**/*.jsx",
-    "**/*.py",
-    "**/*.rs",
-    "**/*.go",
-    "**/*.java",
-    "**/*.md",
-    "**/*.txt",
-    "**/*.json",
-    "**/*.yaml",
-    "**/*.yml",
-    "**/*.toml",
-    "**/*.sh",
-    "**/*.bash",
-    "**/*.zsh",
-    "**/*.css",
-    "**/*.html",
-    "**/*.svg",
-    "**/*.sql",
-    "**/*.graphql",
-    "**/*.c",
-    "**/*.cpp",
-    "**/*.h",
-    "**/*.hpp",
-    "**/*.swift",
-    "**/*.kt",
-    "**/*.rb",
-    "**/*.env.example",
-    "**/*.gitignore",
-    "**/Makefile",
-    "**/Dockerfile",
-    "**/Cargo.toml",
-    "**/package.json",
-    "**/tsconfig.json"
-  ];
-  static DEFAULT_IGNORE = [
-    "**/node_modules/**",
-    "**/.git/**",
-    "**/dist/**",
-    "**/build/**",
-    "**/.next/**",
-    "**/.cache/**",
-    "**/coverage/**",
-    "**/__pycache__/**",
-    "**/target/**",
-    "**/vendor/**",
-    "**/.venv/**",
-    "**/*.lock",
-    "**/package-lock.json",
-    "**/yarn.lock",
-    "**/bun.lockb",
-    "**/*.min.js",
-    "**/*.min.css",
-    "**/*.map",
-    "**/*.png",
-    "**/*.jpg",
-    "**/*.jpeg",
-    "**/*.gif",
-    "**/*.ico",
-    "**/*.webp",
-    "**/*.woff",
-    "**/*.woff2",
-    "**/*.ttf",
-    "**/*.eot",
-    "**/*.mp3",
-    "**/*.mp4",
-    "**/*.wav",
-    "**/*.ogg",
-    "**/*.webm",
-    "**/*.zip",
-    "**/*.tar",
-    "**/*.gz",
-    "**/*.br",
-    "**/*.sqlite",
-    "**/*.db",
-    "**/*.lance/**",
-    "**/*.jsonl",
-    "**/secrets/**",
-    "**/.env"
-  ];
-  /** Add a directory as a source collection for indexing. */
-  async sourcesAdd(rootPath, name, options) {
-    const db = this.sqliteDb;
-    const now = (/* @__PURE__ */ new Date()).toISOString();
-    const includePatterns = JSON.stringify(options?.include || _Crystal.DEFAULT_INCLUDE);
-    const ignorePatterns = JSON.stringify(options?.ignore || _Crystal.DEFAULT_IGNORE);
-    const existing = db.prepare("SELECT * FROM source_collections WHERE name = ?").get(name);
-    if (existing) {
-      throw new Error(`Collection "${name}" already exists. Use sourcesSync() to update it.`);
-    }
-    db.prepare(`
-      INSERT INTO source_collections (name, root_path, glob_patterns, ignore_patterns, created_at)
-      VALUES (?, ?, ?, ?, ?)
-    `).run(name, rootPath, includePatterns, ignorePatterns, now);
-    const row = db.prepare("SELECT * FROM source_collections WHERE name = ?").get(name);
-    return row;
-  }
-  /** Remove a source collection and its file records. Chunks remain in LanceDB. */
-  sourcesRemove(name) {
-    const db = this.sqliteDb;
-    const col = db.prepare("SELECT id FROM source_collections WHERE name = ?").get(name);
-    if (!col) return false;
-    db.prepare("DELETE FROM source_files WHERE collection_id = ?").run(col.id);
-    db.prepare("DELETE FROM source_collections WHERE id = ?").run(col.id);
-    return true;
-  }
-  /** Sync a collection: scan files, detect changes, re-index what changed. */
-  async sourcesSync(name, options) {
-    const db = this.sqliteDb;
-    const startTime = Date.now();
-    const batchSize = options?.batchSize || 20;
-    const col = db.prepare("SELECT * FROM source_collections WHERE name = ?").get(name);
-    if (!col) throw new Error(`Collection "${name}" not found. Add it first with sourcesAdd().`);
-    const includePatterns = JSON.parse(col.glob_patterns);
-    const ignorePatterns = JSON.parse(col.ignore_patterns);
-    const files = this.scanDirectory(col.root_path, includePatterns, ignorePatterns);
-    const existingFiles = /* @__PURE__ */ new Map();
-    const rows = db.prepare("SELECT id, file_path, file_hash FROM source_files WHERE collection_id = ?").all(col.id);
-    for (const row of rows) {
-      existingFiles.set(row.file_path, { id: row.id, file_hash: row.file_hash });
-    }
-    let added = 0;
-    let updated = 0;
-    let removed = 0;
-    let chunksAdded = 0;
-    const now = (/* @__PURE__ */ new Date()).toISOString();
-    const toIndex = [];
-    for (const absPath of files) {
-      const relPath = relative(col.root_path, absPath);
-      let content;
-      try {
-        content = readFileSync(absPath, "utf-8");
-      } catch {
-        continue;
-      }
-      const stat = statSync(absPath);
-      if (stat.size > 500 * 1024) continue;
-      const hash = createHash("sha256").update(content).digest("hex");
-      const existing = existingFiles.get(relPath);
-      if (existing) {
-        existingFiles.delete(relPath);
-        if (existing.file_hash === hash) continue;
-        toIndex.push({ relPath, absPath, hash, size: stat.size, isUpdate: true });
-      } else {
-        toIndex.push({ relPath, absPath, hash, size: stat.size, isUpdate: false });
-      }
-    }
-    if (options?.dryRun) {
-      const newFiles = toIndex.filter((f) => !f.isUpdate).length;
-      const updatedFiles = toIndex.filter((f) => f.isUpdate).length;
-      return {
-        collection: name,
-        added: newFiles,
-        updated: updatedFiles,
-        removed: existingFiles.size,
-        chunks_added: 0,
-        duration_ms: Date.now() - startTime
-      };
-    }
-    for (let i = 0; i < toIndex.length; i += batchSize) {
-      const batch = toIndex.slice(i, i + batchSize);
-      const allChunks = [];
-      for (const file of batch) {
-        const content = readFileSync(file.absPath, "utf-8");
-        const ext = extname(file.absPath);
-        const fileName = basename(file.absPath);
-        const header = `File: ${file.relPath}
-
-`;
-        const textChunks = this.chunkText(header + content, 400, 80);
-        const fileChunks = textChunks.map((text) => ({
-          text,
-          role: "system",
-          source_type: "file",
-          source_id: `file:${name}:${file.relPath}`,
-          agent_id: "system",
-          token_count: Math.ceil(text.length / 4),
-          created_at: now
-        }));
-        allChunks.push(...fileChunks);
-        if (file.isUpdate) {
-          db.prepare(`
-            UPDATE source_files SET file_hash = ?, file_size = ?, chunk_count = ?, last_indexed_at = ?
-            WHERE collection_id = ? AND file_path = ?
-          `).run(file.hash, file.size, fileChunks.length, now, col.id, file.relPath);
-          updated++;
-        } else {
-          db.prepare(`
-            INSERT INTO source_files (collection_id, file_path, file_hash, file_size, chunk_count, last_indexed_at)
-            VALUES (?, ?, ?, ?, ?, ?)
-          `).run(col.id, file.relPath, file.hash, file.size, fileChunks.length, now);
-          added++;
-        }
-      }
-      if (allChunks.length > 0) {
-        const ingested = await this.ingest(allChunks);
-        chunksAdded += ingested;
-      }
-    }
-    for (const [relPath, { id }] of existingFiles) {
-      db.prepare("DELETE FROM source_files WHERE id = ?").run(id);
-      removed++;
-    }
-    const fileCount = db.prepare("SELECT COUNT(*) as count FROM source_files WHERE collection_id = ?").get(col.id).count;
-    const chunkCount = db.prepare("SELECT SUM(chunk_count) as total FROM source_files WHERE collection_id = ?").get(col.id).total || 0;
-    db.prepare("UPDATE source_collections SET file_count = ?, chunk_count = ?, last_sync_at = ? WHERE id = ?").run(fileCount, chunkCount, now, col.id);
-    return {
-      collection: name,
-      added,
-      updated,
-      removed,
-      chunks_added: chunksAdded,
-      duration_ms: Date.now() - startTime
-    };
-  }
-  /** Get status of all source collections. */
-  sourcesStatus() {
-    const db = this.sqliteDb;
-    const collections = db.prepare("SELECT name, root_path, file_count, chunk_count, last_sync_at FROM source_collections").all();
-    const totalFiles = collections.reduce((sum, c) => sum + c.file_count, 0);
-    const totalChunks = collections.reduce((sum, c) => sum + c.chunk_count, 0);
-    return {
-      collections: collections.map((c) => ({
-        name: c.name,
-        root_path: c.root_path,
-        file_count: c.file_count,
-        chunk_count: c.chunk_count,
-        last_sync_at: c.last_sync_at
-      })),
-      total_files: totalFiles,
-      total_chunks: totalChunks
-    };
-  }
-  /** Scan a directory recursively, matching include/ignore patterns. */
-  scanDirectory(rootPath, includePatterns, ignorePatterns) {
-    const results = [];
-    const allowedExtensions = /* @__PURE__ */ new Set();
-    const allowedExactNames = /* @__PURE__ */ new Set();
-    for (const pattern of includePatterns) {
-      const extMatch = pattern.match(/\*\*\/\*(\.\w+)$/);
-      if (extMatch) {
-        allowedExtensions.add(extMatch[1]);
-      }
-      const nameMatch = pattern.match(/\*\*\/([^*]+)$/);
-      if (nameMatch && !nameMatch[1].startsWith("*.")) {
-        allowedExactNames.add(nameMatch[1]);
-      }
-    }
-    const ignoreDirs = /* @__PURE__ */ new Set();
-    for (const pattern of ignorePatterns) {
-      const dirMatch = pattern.match(/\*\*\/([^/*]+)\/\*\*$/);
-      if (dirMatch) {
-        ignoreDirs.add(dirMatch[1]);
-      }
-    }
-    const ignoreFiles = /* @__PURE__ */ new Set();
-    for (const pattern of ignorePatterns) {
-      const fileMatch = pattern.match(/\*\*\/\*(\.\w+)$/);
-      if (fileMatch) {
-        ignoreFiles.add(fileMatch[1]);
-      }
-      const exactMatch = pattern.match(/\*\*\/([^*]+)$/);
-      if (exactMatch && !exactMatch[1].includes("/")) {
-        ignoreFiles.add(exactMatch[1]);
-      }
-    }
-    const walk = (dir) => {
-      let entries;
-      try {
-        entries = readdirSync(dir);
-      } catch {
-        return;
-      }
-      for (const entry of entries) {
-        const fullPath = join(dir, entry);
-        let stat;
-        try {
-          stat = statSync(fullPath);
-        } catch {
-          continue;
-        }
-        if (stat.isDirectory()) {
-          if (ignoreDirs.has(entry)) continue;
-          if (entry.startsWith(".")) continue;
-          walk(fullPath);
-        } else if (stat.isFile()) {
-          const ext = extname(entry);
-          if (ignoreFiles.has(ext)) continue;
-          if (ignoreFiles.has(entry)) continue;
-          if (allowedExtensions.has(ext) || allowedExactNames.has(entry)) {
-            results.push(fullPath);
-          }
-        }
-      }
-    };
-    walk(rootPath);
-    return results;
-  }
-  // ── Cleanup ──
-  close() {
-    this.sqliteDb?.close();
-  }
-};
-function resolveConfig(overrides) {
-  const HOME = process.env.HOME || "";
-  const ldmMemory = join(HOME, ".ldm", "memory");
-  let dataDir = overrides?.dataDir || process.env.CRYSTAL_DATA_DIR;
-  if (!dataDir) {
-    if (existsSync(join(ldmMemory, "crystal.db"))) {
-      dataDir = ldmMemory;
-    } else {
-      const legacyDir = join(HOME, ".openclaw", "memory-crystal");
-      if (existsSync(join(legacyDir, "crystal.db"))) {
-        dataDir = legacyDir;
-      } else {
-        dataDir = ldmMemory;
-      }
-    }
-  }
-  loadEnvFile(join(dataDir, ".env"));
-  const openaiApiKey = overrides?.openaiApiKey || process.env.OPENAI_API_KEY || opRead("OpenAI API", "api key");
-  const googleApiKey = overrides?.googleApiKey || process.env.GOOGLE_API_KEY || opRead("Google AI", "api key");
-  const remoteToken = overrides?.remoteToken || process.env.CRYSTAL_REMOTE_TOKEN || opRead("Memory Crystal Remote", "token");
-  return {
-    dataDir,
-    embeddingProvider: overrides?.embeddingProvider || process.env.CRYSTAL_EMBEDDING_PROVIDER || "openai",
-    openaiApiKey,
-    openaiModel: overrides?.openaiModel || process.env.CRYSTAL_OPENAI_MODEL || "text-embedding-3-small",
-    ollamaHost: overrides?.ollamaHost || process.env.CRYSTAL_OLLAMA_HOST || "http://localhost:11434",
-    ollamaModel: overrides?.ollamaModel || process.env.CRYSTAL_OLLAMA_MODEL || "nomic-embed-text",
-    googleApiKey,
-    googleModel: overrides?.googleModel || process.env.CRYSTAL_GOOGLE_MODEL || "text-embedding-004",
-    remoteUrl: overrides?.remoteUrl || process.env.CRYSTAL_REMOTE_URL,
-    remoteToken
-  };
-}
-function loadEnvFile(path) {
-  if (!existsSync(path)) return;
-  const content = readFileSync(path, "utf8");
-  for (const line of content.split("\n")) {
-    const trimmed = line.trim();
-    if (!trimmed || trimmed.startsWith("#")) continue;
-    const eqIdx = trimmed.indexOf("=");
-    if (eqIdx === -1) continue;
-    const key = trimmed.slice(0, eqIdx).trim();
-    let value = trimmed.slice(eqIdx + 1).trim();
-    if (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'")) {
-      value = value.slice(1, -1);
-    }
-    if (key && !process.env[key]) {
-      process.env[key] = value;
-    }
-  }
-}
-function opRead(item, field) {
-  try {
-    const HOME = process.env.HOME || "";
-    let saTokenPath = join(HOME, ".ldm", "secrets", "op-sa-token");
-    if (!existsSync(saTokenPath)) {
-      saTokenPath = join(HOME, ".openclaw", "secrets", "op-sa-token");
-    }
-    if (!existsSync(saTokenPath)) return void 0;
-    const saToken = readFileSync(saTokenPath, "utf8").trim();
-    return execSync(`op read "op://Agent Secrets/${item}/${field}" 2>/dev/null`, {
-      encoding: "utf8",
-      env: { ...process.env, OP_SERVICE_ACCOUNT_TOKEN: saToken },
-      timeout: 1e4
-    }).trim() || void 0;
-  } catch {
-    return void 0;
-  }
-}
-var RemoteCrystal = class {
-  url;
-  token;
-  constructor(url, token) {
-    this.url = url.replace(/\/$/, "");
-    this.token = token;
-  }
-  async init() {
-    const resp = await fetch(`${this.url}/health`);
-    if (!resp.ok) {
-      throw new Error(`Remote crystal unreachable: ${resp.status}`);
-    }
-  }
-  async request(path, body) {
-    const resp = await fetch(`${this.url}${path}`, {
-      method: body ? "POST" : "GET",
-      headers: {
-        "Authorization": `Bearer ${this.token}`,
-        "Content-Type": "application/json"
-      },
-      ...body ? { body: JSON.stringify(body) } : {}
-    });
-    if (!resp.ok) {
-      const err = await resp.text();
-      throw new Error(`Remote crystal error ${resp.status}: ${err}`);
-    }
-    return resp.json();
-  }
-  async search(query, limit = 5, filter) {
-    const data = await this.request("/search", { query, limit, agent_id: filter?.agent_id });
-    return data.results || [];
-  }
-  async ingest(chunks) {
-    const data = await this.request("/ingest", { chunks });
-    return data.ingested || 0;
-  }
-  async remember(text, category = "fact") {
-    const data = await this.request("/remember", { text, category });
-    return data.id;
-  }
-  forget(memoryId) {
-    return this.request("/forget", { id: memoryId }).then((d) => d.ok);
-  }
-  async status() {
-    const data = await this.request("/status");
-    return {
-      chunks: data.chunks || 0,
-      memories: data.memories || 0,
-      sources: 0,
-      agents: data.agents || [],
-      oldestChunk: data.oldestChunk,
-      newestChunk: data.newestChunk,
-      embeddingProvider: "remote",
-      dataDir: this.url,
-      capturedSessions: data.capturedSessions || 0,
-      latestCapture: data.newestChunk
-    };
-  }
-  // Expose chunkText from a local Crystal instance for cc-hook to use
-  chunkText(text) {
-    const targetChars = 400 * 4;
-    const overlapChars = 80 * 4;
-    if (text.length <= targetChars) return [text];
-    const chunks = [];
-    let start = 0;
-    while (start < text.length) {
-      let end = start + targetChars;
-      if (end >= text.length) {
-        chunks.push(text.slice(start));
-        break;
-      }
-      const paraBreak = text.lastIndexOf("\n\n", end);
-      if (paraBreak > start + targetChars * 0.5) end = paraBreak;
-      else {
-        const sentBreak = text.lastIndexOf(". ", end);
-        if (sentBreak > start + targetChars * 0.5) end = sentBreak + 1;
-      }
-      chunks.push(text.slice(start, end));
-      start = end - overlapChars;
-    }
-    return chunks;
-  }
-};
-function createCrystal(config) {
-  if (config.remoteUrl && config.remoteToken) {
-    return new RemoteCrystal(config.remoteUrl, config.remoteToken);
-  }
-  return new Crystal(config);
-}
-
-export {
-  Crystal,
-  resolveConfig,
-  RemoteCrystal,
-  createCrystal
-};