alvin-bot 4.19.2 → 4.20.0

package/CHANGELOG.md

@@ -2,6 +2,20 @@
 
 All notable changes to Alvin Bot are documented here.
 
+## [4.20.0] — 2026-05-03
+
+### 🚀 Embeddings: JSON → SQLite
+
+**Why.** The vector index `~/.alvin-bot/memory/.embeddings.json` had grown to **146 MB**. Every bot start parsed the whole file (slow boot, large heap), and every reindex iteration rewrote the entire 146 MB blob to disk. With ~3 800 entries the corpus is still small enough that linear-scan cosine similarity is fine, but the JSON serialisation overhead and per-write full-file rewrite were the real cost.
+
+**Change.** New SQLite-backed store at `~/.alvin-bot/memory/.embeddings.db` (table `entries(id, source, text, vector BLOB, indexed_at)` + index on `source`). Vectors live as raw `Float32Array` BLOBs (4 B × 3072 dims = 12 KB each) instead of JSON-encoded Float64 arrays (≈ 24 KB each). Reindexing is per-chunk INSERT/UPDATE inside a single transaction — no full-file rewrite. WAL mode + 256 MB mmap, `synchronous = NORMAL`.
+
+**Migration.** `src/services/embeddings-migration.ts` runs once on boot if `.embeddings.json` exists but `.embeddings.db` does not. Source JSON is renamed to `.embeddings.json.bak-pre-sqlite` after a successful entry-count match (idempotent, safe to re-run). On the maintainer's instance: 146 MB → 49 MB, 3 799 entries copied in 660 ms.
+
+**Files touched.** `src/paths.ts` (new `EMBEDDINGS_DB`), `src/services/embeddings.ts` (full rewrite, drop-in same public surface), `src/services/embeddings-migration.ts` (new), `src/index.ts` (boot hook), `package.json` (deps `better-sqlite3@^12`, `@types/better-sqlite3` dev). Public API unchanged: `searchMemory`, `reindexMemory`, `initEmbeddings`, `getIndexStats` keep their signatures so callers in `engine.ts`, `web-server.ts` etc. don't change.
+
+**Wins.** ~66 % smaller on disk. Bot boot no longer parses a 146 MB JSON. Reindex of a single file is O(log n) DELETE-by-source + transactional INSERTs instead of `JSON.stringify` + `writeFileSync` of the whole index.
+
 ## [4.19.2] — 2026-04-24
 
 ### 🐛 Fix: workspace switch produced "(no response)" format-kaskade; added empty-stream diagnostics

package/dist/index.js

@@ -20,6 +20,17 @@ if (hasLegacyData()) {
 }
 // 3. Seed defaults for any files that don't exist yet (fresh install)
 seedDefaults();
+// 3b. v4.20 — One-shot migration of legacy .embeddings.json → SQLite (.embeddings.db).
+//     Idempotent and safe: source JSON is renamed to .bak-pre-sqlite after success.
+import { shouldMigrateEmbeddingsToSqlite, migrateEmbeddingsToSqlite } from "./services/embeddings-migration.js";
+if (shouldMigrateEmbeddingsToSqlite()) {
+    try {
+        migrateEmbeddingsToSqlite();
+    }
+    catch (err) {
+        console.error("❌ Embeddings migration failed — bot will continue with empty SQLite store, JSON kept:", err);
+    }
+}
 // 3a. v4.12.2 — Audit + repair permissions on sensitive files. On multi-user
 //     systems, files written pre-v4.12.2 may have 0o644 / 0o666 mode — i.e.
 //     readable by other users on the same machine. This routine chmod-repairs

package/dist/paths.js

@@ -55,8 +55,11 @@ export const PROJECTS_MEMORY_DIR = resolve(DATA_DIR, "memory", "projects");
  *  name, purpose, cwd, color, emoji, and an optional system prompt body.
  *  See src/services/workspaces.ts for the loader and matcher. */
 export const WORKSPACES_DIR = resolve(DATA_DIR, "workspaces");
-/** memory/.embeddings.json — Vector index */
+/** memory/.embeddings.json — Legacy JSON vector index. Read on first SQLite migration only;
+ *  active code path is EMBEDDINGS_DB. */
 export const EMBEDDINGS_IDX = resolve(DATA_DIR, "memory", ".embeddings.json");
+/** memory/.embeddings.db — SQLite vector store (replaces .embeddings.json since v4.20). */
+export const EMBEDDINGS_DB = resolve(DATA_DIR, "memory", ".embeddings.db");
 /** users/ — User profiles and per-user memory */
 export const USERS_DIR = resolve(DATA_DIR, "users");
 /** data/ — Runtime control data */

package/dist/services/embeddings-migration.js

@@ -0,0 +1,114 @@
+/**
+ * One-shot migration from legacy .embeddings.json → SQLite .embeddings.db.
+ *
+ * Triggered on startup if .embeddings.json exists but .embeddings.db does not.
+ * Idempotent: skips silently if the DB is already populated.
+ *
+ * Safety:
+ *  - Source JSON is renamed to .embeddings.json.bak-pre-sqlite (kept on disk).
+ *  - Entry counts are compared after import; mismatch → throw, leaving the bak
+ *    file in place for manual recovery.
+ */
+import fs from "fs";
+import path from "path";
+import Database from "better-sqlite3";
+import { EMBEDDINGS_IDX, EMBEDDINGS_DB } from "../paths.js";
+function vectorToBlob(v) {
+    const f32 = new Float32Array(v);
+    return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
+}
+export function shouldMigrateEmbeddingsToSqlite() {
+    return fs.existsSync(EMBEDDINGS_IDX) && !fs.existsSync(EMBEDDINGS_DB);
+}
+/**
+ * Run the migration. Returns the entry count migrated, or null if skipped.
+ */
+export function migrateEmbeddingsToSqlite() {
+    if (!shouldMigrateEmbeddingsToSqlite())
+        return null;
+    const t0 = Date.now();
+    const sourceSize = fs.statSync(EMBEDDINGS_IDX).size;
+    console.log(`📦 Migrating embeddings JSON (${(sourceSize / 1024 / 1024).toFixed(0)} MB) → SQLite...`);
+    const raw = fs.readFileSync(EMBEDDINGS_IDX, "utf-8");
+    let legacy;
+    try {
+        legacy = JSON.parse(raw);
+    }
+    catch (err) {
+        console.error("⚠️ Embeddings migration: source JSON is corrupt — skipping.", err);
+        return null;
+    }
+    fs.mkdirSync(path.dirname(EMBEDDINGS_DB), { recursive: true });
+    const db = new Database(EMBEDDINGS_DB);
+    try {
+        db.pragma("journal_mode = WAL");
+        db.pragma("synchronous = NORMAL");
+        db.exec(`
+      CREATE TABLE IF NOT EXISTS meta (
+        key   TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS file_mtimes (
+        source   TEXT PRIMARY KEY,
+        mtime_ms REAL NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS entries (
+        id         TEXT PRIMARY KEY,
+        source     TEXT NOT NULL,
+        text       TEXT NOT NULL,
+        vector     BLOB NOT NULL,
+        indexed_at INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_entries_source ON entries(source);
+    `);
+        const setMeta = db.prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value");
+        setMeta.run("model", legacy.model);
+        setMeta.run("schemaVersion", "1");
+        setMeta.run("lastReindex", String(legacy.lastReindex));
+        setMeta.run("migratedFromJson", String(Date.now()));
+        const insMtime = db.prepare("INSERT INTO file_mtimes (source, mtime_ms) VALUES (?, ?) ON CONFLICT(source) DO UPDATE SET mtime_ms = excluded.mtime_ms");
+        const writeMtimes = db.transaction((rows) => {
+            for (const [s, m] of rows)
+                insMtime.run(s, m);
+        });
+        writeMtimes(Object.entries(legacy.fileMtimes ?? {}));
+        const insEntry = db.prepare("INSERT INTO entries (id, source, text, vector, indexed_at) VALUES (?, ?, ?, ?, ?)");
+        const writeEntries = db.transaction((rows) => {
+            for (const e of rows) {
+                if (!Array.isArray(e.vector) || e.vector.length === 0)
+                    continue;
+                insEntry.run(e.id, e.source, e.text, vectorToBlob(e.vector), e.indexedAt);
+            }
+        });
+        writeEntries(legacy.entries ?? []);
+        const written = db.prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+        const expected = (legacy.entries ?? []).filter(e => Array.isArray(e.vector) && e.vector.length > 0).length;
+        if (written !== expected) {
+            throw new Error(`Entry-count mismatch after migration: expected ${expected}, got ${written}`);
+        }
+        db.close();
+        // Move source JSON aside so we never re-migrate.
+        const bak = `${EMBEDDINGS_IDX}.bak-pre-sqlite`;
+        try {
+            fs.renameSync(EMBEDDINGS_IDX, bak);
+        }
+        catch (err) {
+            console.warn("⚠️ Could not rename source JSON:", err);
+        }
+        const targetSize = fs.statSync(EMBEDDINGS_DB).size;
+        const dt = Date.now() - t0;
+        console.log(`✅ Embeddings migrated: ${written} entries, ${(sourceSize / 1024 / 1024).toFixed(0)} MB JSON → ${(targetSize / 1024 / 1024).toFixed(0)} MB SQLite in ${dt} ms`);
+        return { entries: written, sourceMb: sourceSize / 1024 / 1024, targetMb: targetSize / 1024 / 1024 };
+    }
+    catch (err) {
+        db.close();
+        // Remove half-written DB so the next boot retries cleanly.
+        try {
+            fs.unlinkSync(EMBEDDINGS_DB);
+        }
+        catch {
+            /* nothing to clean */
+        }
+        throw err;
+    }
+}

package/dist/services/embeddings.js

@@ -1,31 +1,116 @@
 /**
  * Embeddings Service — Vector-based semantic memory search.
  *
- * Uses Google's text-embedding-004 model for generating embeddings.
- * Stores embeddings in a local JSON index file for fast cosine similarity search.
+ * Uses Google's gemini-embedding-001 model for generating embeddings.
+ * Stores embeddings in a SQLite database (.embeddings.db) — replaces the
+ * older .embeddings.json index since v4.20. The migration runs once
+ * automatically on startup (see src/migrate.ts).
  *
  * Architecture:
- * - Each memory entry (paragraph/section) gets an embedding vector
- * - Vectors are stored in docs/memory/.embeddings.json
- * - On query, the search text is embedded and compared via cosine similarity
- * - Top-K results returned with similarity scores
+ * - Each memory entry (paragraph/section) gets a 3072-dim Float32 vector.
+ * - Vectors are stored as raw BLOB (4 bytes × 3072 = 12 KB each) instead of
+ *   JSON-encoded Float64 arrays (~24 KB each) — halves disk footprint.
+ * - Cosine similarity runs in-memory: SQLite has no native vector ops, but
+ *   reading the BLOBs is mmap-cheap and JS does the dot product fast enough
+ *   for the current corpus (a few thousand entries).
+ * - Reindexing is per-chunk INSERT/UPDATE — no full-file rewrite.
  */
 import fs from "fs";
 import path from "path";
 import { resolve } from "path";
-import { config } from "../config.js";
 import os from "os";
-import { MEMORY_DIR, MEMORY_FILE, EMBEDDINGS_IDX as INDEX_FILE } from "../paths.js";
+import Database from "better-sqlite3";
+import { config } from "../config.js";
+import { MEMORY_DIR, MEMORY_FILE, EMBEDDINGS_DB } from "../paths.js";
 import { ASSETS_DIR, ASSETS_INDEX_MD } from "../paths.js";
 // Hub memory directory (Claude Hub — read-only, additional context)
 const HUB_MEMORY_DIR = resolve(os.homedir(), ".claude", "hub", "MEMORY");
-// ── Google Embeddings API ───────────────────────────────
+// ── Constants ───────────────────────────────────────────
 const EMBEDDING_MODEL = "gemini-embedding-001";
 const EMBEDDING_DIMENSION = 3072;
-/**
- * Get embeddings for one or more texts via Google's API.
- * Batches up to 100 texts per request.
- */
+const SCHEMA_VERSION = "1";
+// ── Vector encoding (Float32Array ↔ Buffer) ─────────────
+function vectorToBlob(v) {
+    const f32 = new Float32Array(v);
+    // Buffer.from(arrayBuffer, byteOffset, length) preserves the underlying memory.
+    return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
+}
+function blobToVector(b) {
+    // Buffers from better-sqlite3 own their memory and may not be aligned to 4 bytes.
+    // Copying into a fresh Float32Array guarantees alignment.
+    const f32 = new Float32Array(b.byteLength / 4);
+    const dv = new DataView(b.buffer, b.byteOffset, b.byteLength);
+    for (let i = 0; i < f32.length; i++) {
+        f32[i] = dv.getFloat32(i * 4, true /* little-endian */);
+    }
+    return f32;
+}
+// ── DB lifecycle ────────────────────────────────────────
+let dbInstance = null;
+function db() {
+    if (dbInstance)
+        return dbInstance;
+    // Ensure directory exists (handles fresh installs).
+    fs.mkdirSync(path.dirname(EMBEDDINGS_DB), { recursive: true });
+    dbInstance = new Database(EMBEDDINGS_DB);
+    dbInstance.pragma("journal_mode = WAL");
+    dbInstance.pragma("synchronous = NORMAL");
+    dbInstance.pragma("temp_store = MEMORY");
+    dbInstance.pragma("mmap_size = 268435456"); // 256 MB
+    dbInstance.exec(`
+    CREATE TABLE IF NOT EXISTS meta (
+      key   TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    );
+    CREATE TABLE IF NOT EXISTS file_mtimes (
+      source   TEXT PRIMARY KEY,
+      mtime_ms REAL NOT NULL
+    );
+    CREATE TABLE IF NOT EXISTS entries (
+      id         TEXT PRIMARY KEY,
+      source     TEXT NOT NULL,
+      text       TEXT NOT NULL,
+      vector     BLOB NOT NULL,
+      indexed_at INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_entries_source ON entries(source);
+  `);
+    // Initialise meta if absent.
+    const set = dbInstance.prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO NOTHING");
+    set.run("model", EMBEDDING_MODEL);
+    set.run("schemaVersion", SCHEMA_VERSION);
+    return dbInstance;
+}
+/** Close handle (used by tests / shutdown). */
+export function closeEmbeddingsDb() {
+    if (dbInstance) {
+        dbInstance.close();
+        dbInstance = null;
+    }
+}
+// ── Meta helpers ────────────────────────────────────────
+function getMeta(key) {
+    const row = db().prepare("SELECT value FROM meta WHERE key = ?").get(key);
+    return row?.value ?? null;
+}
+function setMeta(key, value) {
+    db()
+        .prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value")
+        .run(key, value);
+}
+function getFileMtimes() {
+    const rows = db().prepare("SELECT source, mtime_ms FROM file_mtimes").all();
+    const out = {};
+    for (const r of rows)
+        out[r.source] = r.mtime_ms;
+    return out;
+}
+function setFileMtime(source, mtimeMs) {
+    db()
+        .prepare("INSERT INTO file_mtimes (source, mtime_ms) VALUES (?, ?) ON CONFLICT(source) DO UPDATE SET mtime_ms = excluded.mtime_ms")
+        .run(source, mtimeMs);
+}
+// ── Google Embeddings API ───────────────────────────────
 async function getEmbeddings(texts) {
     const apiKey = config.apiKeys.google;
     if (!apiKey) {
@@ -50,16 +135,13 @@ async function getEmbeddings(texts) {
             const err = await response.text();
             throw new Error(`Embedding API error: ${response.status} — ${err}`);
         }
-        const data = await response.json();
+        const data = (await response.json());
         for (const emb of data.embeddings) {
             results.push(emb.values);
         }
     }
     return results;
 }
-/**
- * Get embedding for a single query text.
- */
 async function getQueryEmbedding(text) {
     const apiKey = config.apiKeys.google;
     if (!apiKey) {
@@ -78,11 +160,11 @@ async function getQueryEmbedding(text) {
         const err = await response.text();
         throw new Error(`Embedding API error: ${response.status} — ${err}`);
     }
-    const data = await response.json();
+    const data = (await response.json());
     return data.embedding.values;
 }
 // ── Vector Math ─────────────────────────────────────────
-function cosineSimilarity(a, b) {
+function cosineSimilarityF32(a, b) {
     if (a.length !== b.length)
         return 0;
     let dotProduct = 0;
@@ -97,20 +179,13 @@ function cosineSimilarity(a, b) {
     return denom === 0 ? 0 : dotProduct / denom;
 }
 // ── Text Chunking ───────────────────────────────────────
-/**
- * Split a markdown file into meaningful chunks.
- * Splits on ## headers, keeping each section as a chunk.
- * Falls back to paragraph splitting for files without headers.
- */
 function chunkMarkdown(content, source) {
     const chunks = [];
-    // Split on ## headers
     const sections = content.split(/^(?=## )/gm);
     for (let i = 0; i < sections.length; i++) {
         const section = sections[i].trim();
         if (!section || section.length < 20)
-            continue; // Skip tiny sections
-        // If section is too long (>1000 chars), split into paragraphs
+            continue;
         if (section.length > 1000) {
             const paragraphs = section.split(/\n\n+/);
             let currentChunk = "";
@@ -142,51 +217,7 @@ function chunkMarkdown(content, source) {
     }
     return chunks;
 }
-// ── Index Management ────────────────────────────────────
-// In-memory cache for the embedding index. Without this, every query would
-// re-read and re-parse the on-disk index (can be 100+ MB, making searchMemory
-// the slowest step in a message turn). We keep the parsed object and invalidate
-// via mtime check — so external reindexers are still picked up.
-let indexCache = null;
-let indexCacheMtime = 0;
-function loadIndex() {
-    try {
-        const st = fs.statSync(INDEX_FILE);
-        if (indexCache && st.mtimeMs === indexCacheMtime) {
-            return indexCache;
-        }
-        const raw = fs.readFileSync(INDEX_FILE, "utf-8");
-        indexCache = JSON.parse(raw);
-        indexCacheMtime = st.mtimeMs;
-        return indexCache;
-    }
-    catch {
-        // File missing or unparseable — return an empty index and don't cache it
-        // (next call will retry, so a freshly-written index gets picked up).
-        return {
-            model: EMBEDDING_MODEL,
-            lastReindex: 0,
-            fileMtimes: {},
-            entries: [],
-        };
-    }
-}
-function saveIndex(index) {
-    fs.writeFileSync(INDEX_FILE, JSON.stringify(index));
-    // Refresh cache immediately so the next loadIndex() sees the new state
-    // without a disk round-trip.
-    indexCache = index;
-    try {
-        indexCacheMtime = fs.statSync(INDEX_FILE).mtimeMs;
-    }
-    catch {
-        indexCacheMtime = Date.now();
-    }
-}
-/**
- * Recursively walk a directory, returning file paths.
- * Skips INDEX.json and INDEX.md at the directory root.
- */
+// ── Indexable file discovery ────────────────────────────
 function walkAssetDir(dir) {
     const results = [];
     function walk(currentDir) {
@@ -213,17 +244,11 @@ function walkAssetDir(dir) {
     return results;
 }
 const TEXT_EXTENSIONS = new Set([".md", ".html", ".txt", ".css", ".ts"]);
-/**
- * Get all files that should be indexed — memories + text-based assets.
- */
 function getIndexableFiles() {
     const files = [];
-    // ── Memories (existing) ───────────────────────────────
-    // Alvin-Bot MEMORY.md
     if (fs.existsSync(MEMORY_FILE)) {
         files.push({ path: MEMORY_FILE, relativePath: "MEMORY.md" });
     }
-    // Alvin-Bot daily logs
     if (fs.existsSync(MEMORY_DIR)) {
         const entries = fs.readdirSync(MEMORY_DIR);
         for (const entry of entries) {
@@ -235,7 +260,6 @@ function getIndexableFiles() {
             }
         }
     }
-    // Hub memories (~/.claude/hub/MEMORY/) — Claude Hub knowledge base
     if (fs.existsSync(HUB_MEMORY_DIR)) {
         try {
             const entries = fs.readdirSync(HUB_MEMORY_DIR);
@@ -248,14 +272,13 @@ function getIndexableFiles() {
                 }
             }
         }
-        catch { /* Hub not available — skip */ }
+        catch {
+            /* Hub not available — skip */
+        }
     }
-    // ── Assets (new) ──────────────────────────────────────
-    // Asset INDEX.md — compact summary of all assets
     if (fs.existsSync(ASSETS_INDEX_MD)) {
         files.push({ path: ASSETS_INDEX_MD, relativePath: "assets/INDEX.md" });
     }
-    // Text-based asset files (HTML, MD, TXT, CSS, TS)
     if (fs.existsSync(ASSETS_DIR)) {
         for (const entry of walkAssetDir(ASSETS_DIR)) {
             if (TEXT_EXTENSIONS.has(path.extname(entry.name))) {
@@ -268,120 +291,133 @@ function getIndexableFiles() {
     }
     return files;
 }
-/**
- * Check which files need reindexing (new or modified).
- */
-function getStaleFiles(index) {
+function getStaleFiles() {
     const allFiles = getIndexableFiles();
+    const known = getFileMtimes();
     const stale = [];
     for (const file of allFiles) {
         try {
-            const stat = fs.statSync(file.path);
-            const mtime = stat.mtimeMs;
-            if (!index.fileMtimes[file.relativePath] || index.fileMtimes[file.relativePath] < mtime) {
+            const mtime = fs.statSync(file.path).mtimeMs;
+            if (!known[file.relativePath] || known[file.relativePath] < mtime) {
                 stale.push(file);
             }
         }
         catch {
-            // File disappeared — skip
+            /* file disappeared */
         }
     }
     return stale;
 }
 // ── Public API ──────────────────────────────────────────
-/**
- * Reindex all memory files (or just stale ones).
- * Returns number of chunks indexed.
- */
 export async function reindexMemory(force = false) {
-    const index = loadIndex();
-    const filesToIndex = force ? getIndexableFiles() : getStaleFiles(index);
+    const filesToIndex = force ? getIndexableFiles() : getStaleFiles();
     if (filesToIndex.length === 0) {
-        return { indexed: 0, total: index.entries.length };
+        const total = db().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+        return { indexed: 0, total };
     }
-    // Remove old entries for files being reindexed
-    const reindexSources = new Set(filesToIndex.map(f => f.relativePath));
-    index.entries = index.entries.filter(e => !reindexSources.has(e.source));
-    // Chunk all files
+    // Drop existing entries for files being reindexed (per-source DELETE is O(log n) thanks to idx).
+    const delStmt = db().prepare("DELETE FROM entries WHERE source = ?");
+    const dropOld = db().transaction((sources) => {
+        for (const s of sources)
+            delStmt.run(s);
+    });
+    dropOld(filesToIndex.map(f => f.relativePath));
+    // Chunk all files.
     const allChunks = [];
     for (const file of filesToIndex) {
         try {
             const content = fs.readFileSync(file.path, "utf-8");
             const chunks = chunkMarkdown(content, file.relativePath);
+            const mtime = fs.statSync(file.path).mtimeMs;
             for (const chunk of chunks) {
-                allChunks.push({ ...chunk, source: file.relativePath });
+                allChunks.push({ ...chunk, source: file.relativePath, mtime });
             }
-            // Update mtime
-            const stat = fs.statSync(file.path);
-            index.fileMtimes[file.relativePath] = stat.mtimeMs;
         }
         catch (err) {
             console.error(`Failed to chunk ${file.relativePath}:`, err);
         }
     }
     if (allChunks.length === 0) {
-        saveIndex(index);
-        return { indexed: 0, total: index.entries.length };
+        // Even with zero chunks, keep mtimes in sync so we don't re-walk on next run.
+        const updMtime = db().transaction((files) => {
+            for (const f of files) {
+                try {
+                    setFileMtime(f.relativePath, fs.statSync(f.path).mtimeMs);
+                }
+                catch {
+                    /* file disappeared */
+                }
+            }
+        });
+        updMtime(filesToIndex);
+        const total = db().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+        return { indexed: 0, total };
     }
-    // Get embeddings for all chunks
+    // Get embeddings for all chunks (network).
     const texts = allChunks.map(c => c.text);
     const vectors = await getEmbeddings(texts);
-    // Add to index
-    for (let i = 0; i < allChunks.length; i++) {
-        index.entries.push({
-            id: allChunks[i].id,
-            source: allChunks[i].source,
-            text: allChunks[i].text,
-            vector: vectors[i],
-            indexedAt: Date.now(),
-        });
-    }
-    index.lastReindex = Date.now();
-    saveIndex(index);
-    return { indexed: allChunks.length, total: index.entries.length };
+    // Single transaction for all writes.
+    const insertStmt = db().prepare("INSERT INTO entries (id, source, text, vector, indexed_at) VALUES (?, ?, ?, ?, ?) " +
+        "ON CONFLICT(id) DO UPDATE SET source=excluded.source, text=excluded.text, vector=excluded.vector, indexed_at=excluded.indexed_at");
+    const writeAll = db().transaction((rows) => {
+        for (const r of rows) {
+            insertStmt.run(r.id, r.source, r.text, r.vector, r.indexedAt);
+        }
+    });
+    const now = Date.now();
+    writeAll(allChunks.map((c, i) => ({
+        id: c.id,
+        source: c.source,
+        text: c.text,
+        vector: vectorToBlob(vectors[i]),
+        indexedAt: now,
+    })));
+    // Update mtimes for the files we just (re-)indexed.
+    const updMtime = db().transaction((files) => {
+        for (const f of files) {
+            try {
+                setFileMtime(f.relativePath, fs.statSync(f.path).mtimeMs);
+            }
+            catch {
+                /* file disappeared */
+            }
+        }
+    });
+    updMtime(filesToIndex);
+    setMeta("lastReindex", String(now));
+    const total = db().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+    return { indexed: allChunks.length, total };
 }
-/**
- * Semantic search across all indexed memory.
- * Returns top-K results sorted by similarity.
- */
 export async function searchMemory(query, topK = 5, minScore = 0.3) {
-    const index = loadIndex();
-    if (index.entries.length === 0) {
-        // Auto-index if empty
+    // Auto-index if empty.
+    const total = db().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+    if (total === 0) {
         await reindexMemory();
-        // Reload
-        const reloaded = loadIndex();
-        if (reloaded.entries.length === 0)
+        const after = db().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+        if (after === 0)
             return [];
     }
-    // Get query embedding
-    const queryVector = await getQueryEmbedding(query);
-    // Calculate similarities
-    const scored = index.entries.map(entry => ({
-        text: entry.text,
-        source: entry.source,
-        score: cosineSimilarity(queryVector, entry.vector),
-    }));
-    // Sort by score descending, filter by minScore, take topK
-    return scored
-        .filter(r => r.score >= minScore)
-        .sort((a, b) => b.score - a.score)
-        .slice(0, topK);
+    const queryVector = Float32Array.from(await getQueryEmbedding(query));
+    const rows = db().prepare("SELECT id, source, text, vector FROM entries").all();
+    const scored = [];
+    for (const row of rows) {
+        const v = blobToVector(row.vector);
+        const score = cosineSimilarityF32(queryVector, v);
+        if (score >= minScore) {
+            scored.push({ text: row.text, source: row.source, score });
+        }
+    }
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, topK);
 }
-/**
- * Get index stats for /status.
- */
-/**
- * Auto-reindex on startup. Indexes only stale/new files (incremental).
- * Runs in background — does not block bot startup.
- */
 export async function initEmbeddings() {
     try {
-        const stale = getStaleFiles(loadIndex());
+        db(); // Open & migrate schema.
+        const stale = getStaleFiles();
         if (stale.length === 0) {
-            const idx = loadIndex();
-            if (idx.entries.length > 0)
-                return; // Already indexed, nothing stale
+            const total = db().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+            if (total > 0)
+                return;
         }
         const result = await reindexMemory();
         if (result.indexed > 0) {
@@ -389,21 +425,26 @@ export async function initEmbeddings() {
         }
     }
     catch (err) {
-        // Non-fatal — bot works without embeddings
         console.warn("⚠️ Embeddings init failed:", err instanceof Error ? err.message : err);
     }
 }
 export function getIndexStats() {
-    const index = loadIndex();
+    let entries = 0;
+    let files = 0;
+    let lastReindex = 0;
     let sizeBytes = 0;
     try {
-        sizeBytes = fs.statSync(INDEX_FILE).size;
+        entries = db().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+        files = db().prepare("SELECT COUNT(*) AS c FROM file_mtimes").get().c;
+        const meta = getMeta("lastReindex");
+        if (meta)
+            lastReindex = Number(meta);
+        sizeBytes = fs.statSync(EMBEDDINGS_DB).size;
+    }
+    catch {
+        /* DB not yet initialised */
     }
-    catch { /* empty */ }
-    return {
-        entries: index.entries.length,
-        files: Object.keys(index.fileMtimes).length,
-        lastReindex: index.lastReindex,
-        sizeBytes,
-    };
+    return { entries, files, lastReindex, sizeBytes };
 }
+// ── Re-export embedding dim for tests / debugging ──────
+export { EMBEDDING_DIMENSION, EMBEDDING_MODEL };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "alvin-bot",
-  "version": "4.19.2",
-  "description": "Alvin Bot \u2014 Your personal AI agent on Telegram, WhatsApp, Discord, Signal, and Web.",
+  "version": "4.20.0",
+  "description": "Alvin Bot — Your personal AI agent on Telegram, WhatsApp, Discord, Signal, and Web.",
   "type": "module",
   "main": "dist/index.js",
   "bin": {
@@ -170,6 +170,7 @@
     "@types/node": "^22.0.0",
     "@types/ws": "^8.18.1",
     "@whiskeysockets/baileys": "^6.7.21",
+    "better-sqlite3": "^12.9.0",
     "dotenv": "^16.4.0",
     "electron-updater": "^6.8.3",
     "grammy": "^1.30.0",
@@ -181,6 +182,7 @@
     "ws": "^8.19.0"
   },
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
     "@vitest/ui": "^4.1.4",
     "electron": "^35.7.5",
     "electron-builder": "^26.8.1",