alvin-bot 4.19.2 → 4.20.1

package/CHANGELOG.md

@@ -2,6 +2,35 @@
 
 All notable changes to Alvin Bot are documented here.
 
+## [4.20.1] — 2026-05-03
+
+### 🛡️ Hardening for the v4.20.0 SQLite migration
+
+The v4.20 migration is fully automatic on first start, but a few things could go wrong on user installations that the maintainer instance never hits. v4.20.1 plugs each of them.
+
+- **Lazy native binary load.** `better-sqlite3` is now `require()`-d inside `embeddings.ts`, not at module import time. If the prebuilt isn't available for the user's platform and a build-from-source fails (exotic Node version, missing toolchain, glibc mismatch), the bot logs a single clear warning with the exact rebuild command, and **keeps running** — only semantic memory search is disabled until the user fixes their install. Previously this would have crashed bot startup.
+- **Pre-flight disk-space check.** Migration refuses to start unless the volume holding `~/.alvin-bot/memory/` has at least 2× the source JSON's size free (covers source + target + WAL during the transaction). Skipped migration leaves the JSON intact for retry on the next boot once space is free.
+- **Progress logging.** On indexes larger than ~5 000 entries, the migration logs `…migrated N / M entries (P %)` every 5 000 rows so the user can see it isn't stuck.
+- **Corrupt JSON recovery.** If `JSON.parse` of `.embeddings.json` throws, the file is moved aside to `.embeddings.json.broken.<timestamp>` and the next bot start treats this as a fresh install (rebuild-from-source on first search). No more boot-loop on a damaged index.
+- **`alvin-bot doctor` shows memory health.** New "Memory:" section reports: native binary loadable, vector-store entry count + size, or — for not-yet-migrated installs — the legacy JSON's size and a hint that the next start will migrate.
+- **Cleanup on failed migration.** WAL/SHM sidecars are removed alongside the half-written `.embeddings.db` so the next attempt starts from a clean slate.
+
+No schema or API changes — drop-in over v4.20.0.
+
+## [4.20.0] — 2026-05-03
+
+### 🚀 Embeddings: JSON → SQLite
+
+**Why.** The vector index `~/.alvin-bot/memory/.embeddings.json` had grown to **146 MB**. Every bot start parsed the whole file (slow boot, large heap), and every reindex iteration rewrote the entire 146 MB blob to disk. With ~3 800 entries the corpus is still small enough that linear-scan cosine similarity is fine, but the JSON serialisation overhead and per-write full-file rewrite were the real cost.
+
+**Change.** New SQLite-backed store at `~/.alvin-bot/memory/.embeddings.db` (table `entries(id, source, text, vector BLOB, indexed_at)` + index on `source`). Vectors live as raw `Float32Array` BLOBs (4 B × 3072 dims = 12 KB each) instead of JSON-encoded Float64 arrays (≈ 24 KB each). Reindexing is per-chunk INSERT/UPDATE inside a single transaction — no full-file rewrite. WAL mode + 256 MB mmap, `synchronous = NORMAL`.
+
+**Migration.** `src/services/embeddings-migration.ts` runs once on boot if `.embeddings.json` exists but `.embeddings.db` does not. Source JSON is renamed to `.embeddings.json.bak-pre-sqlite` after a successful entry-count match (idempotent, safe to re-run). On the maintainer's instance: 146 MB → 49 MB, 3 799 entries copied in 660 ms.
+
+**Files touched.** `src/paths.ts` (new `EMBEDDINGS_DB`), `src/services/embeddings.ts` (full rewrite, drop-in same public surface), `src/services/embeddings-migration.ts` (new), `src/index.ts` (boot hook), `package.json` (deps `better-sqlite3@^12`, `@types/better-sqlite3` dev). Public API unchanged: `searchMemory`, `reindexMemory`, `initEmbeddings`, `getIndexStats` keep their signatures so callers in `engine.ts`, `web-server.ts` etc. don't change.
+
+**Wins.** ~66 % smaller on disk. Bot boot no longer parses a 146 MB JSON. Reindex of a single file is O(log n) DELETE-by-source + transactional INSERTs instead of `JSON.stringify` + `writeFileSync` of the whole index.
+
 ## [4.19.2] — 2026-04-24
 
 ### 🐛 Fix: workspace switch produced "(no response)" format-kaskade; added empty-stream diagnostics

package/bin/cli.js

@@ -17,7 +17,7 @@
  */
 
 import { createInterface } from "readline";
-import { existsSync, writeFileSync, readFileSync, mkdirSync, copyFileSync, readdirSync } from "fs";
+import { existsSync, writeFileSync, readFileSync, mkdirSync, copyFileSync, readdirSync, statSync } from "fs";
 import { resolve, join } from "path";
 import { homedir } from "os";
 import { execSync } from "child_process";
@@ -1361,6 +1361,52 @@ async function doctor() {
     console.log(`  ❌ ALLOWED_USERS not set (nobody can message the bot)`);
   }
 
+  // ── Memory (semantic search backend) ──
+  console.log("\n  Memory:");
+  const embJson = resolve(DATA_DIR, "memory", ".embeddings.json");
+  const embDb = resolve(DATA_DIR, "memory", ".embeddings.db");
+  const embBakSqlite = resolve(DATA_DIR, "memory", ".embeddings.json.bak-pre-sqlite");
+
+  // better-sqlite3 native binary loadable?
+  let sqliteOk = false;
+  let sqliteErr = "";
+  try {
+    const req = (await import("module")).createRequire(import.meta.url);
+    req("better-sqlite3");
+    sqliteOk = true;
+  } catch (err) {
+    sqliteErr = err instanceof Error ? err.message : String(err);
+  }
+  if (sqliteOk) {
+    console.log(`  ✅ better-sqlite3 native binary loadable`);
+  } else {
+    console.log(`  ❌ better-sqlite3 native binary not loadable — semantic search disabled`);
+    console.log(`     Fix: cd $(npm root -g)/alvin-bot && npm rebuild better-sqlite3`);
+    console.log(`     Detail: ${sqliteErr.split("\n")[0]}`);
+  }
+
+  if (sqliteOk && existsSync(embDb)) {
+    try {
+      const req = (await import("module")).createRequire(import.meta.url);
+      const Database = req("better-sqlite3");
+      const db = new Database(embDb, { readonly: true });
+      const entries = db.prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+      const files = db.prepare("SELECT COUNT(*) AS c FROM file_mtimes").get().c;
+      const sizeMb = (statSync(embDb).size / 1024 / 1024).toFixed(0);
+      db.close();
+      console.log(`  ✅ Vector store: ${entries} entries across ${files} sources (${sizeMb} MB SQLite)`);
+    } catch (err) {
+      console.log(`  ⚠️  Vector store exists but unreadable: ${err.message}`);
+    }
+  } else if (existsSync(embJson)) {
+    const sizeMb = (statSync(embJson).size / 1024 / 1024).toFixed(0);
+    console.log(`  ⚠️  Legacy JSON index found (${sizeMb} MB) — will auto-migrate to SQLite on next bot start`);
+  } else if (existsSync(embBakSqlite)) {
+    console.log(`  ✅ Migration to SQLite already done (legacy JSON kept as .bak-pre-sqlite)`);
+  } else {
+    console.log(`  ℹ️  No vector store yet — will be built on first message`);
+  }
+
   // ── Extras ──
   console.log("\n  Extras:");
 

package/dist/index.js

@@ -20,6 +20,17 @@ if (hasLegacyData()) {
 }
 // 3. Seed defaults for any files that don't exist yet (fresh install)
 seedDefaults();
+// 3b. v4.20 — One-shot migration of legacy .embeddings.json → SQLite (.embeddings.db).
+//     Idempotent and safe: source JSON is renamed to .bak-pre-sqlite after success.
+import { shouldMigrateEmbeddingsToSqlite, migrateEmbeddingsToSqlite } from "./services/embeddings-migration.js";
+if (shouldMigrateEmbeddingsToSqlite()) {
+    try {
+        migrateEmbeddingsToSqlite();
+    }
+    catch (err) {
+        console.error("❌ Embeddings migration failed — bot will continue with empty SQLite store, JSON kept:", err);
+    }
+}
 // 3a. v4.12.2 — Audit + repair permissions on sensitive files. On multi-user
 //     systems, files written pre-v4.12.2 may have 0o644 / 0o666 mode — i.e.
 //     readable by other users on the same machine. This routine chmod-repairs

package/dist/paths.js

@@ -55,8 +55,11 @@ export const PROJECTS_MEMORY_DIR = resolve(DATA_DIR, "memory", "projects");
  *  name, purpose, cwd, color, emoji, and an optional system prompt body.
  *  See src/services/workspaces.ts for the loader and matcher. */
 export const WORKSPACES_DIR = resolve(DATA_DIR, "workspaces");
-/** memory/.embeddings.json — Vector index */
+/** memory/.embeddings.json — Legacy JSON vector index. Read on first SQLite migration only;
+ *  active code path is EMBEDDINGS_DB. */
 export const EMBEDDINGS_IDX = resolve(DATA_DIR, "memory", ".embeddings.json");
+/** memory/.embeddings.db — SQLite vector store (replaces .embeddings.json since v4.20). */
+export const EMBEDDINGS_DB = resolve(DATA_DIR, "memory", ".embeddings.db");
 /** users/ — User profiles and per-user memory */
 export const USERS_DIR = resolve(DATA_DIR, "users");
 /** data/ — Runtime control data */

package/dist/services/embeddings-migration.js

@@ -0,0 +1,193 @@
+/**
+ * One-shot migration from legacy .embeddings.json → SQLite .embeddings.db.
+ *
+ * Triggered on startup if .embeddings.json exists but .embeddings.db does not.
+ * Idempotent: skips silently if the DB is already populated.
+ *
+ * Hardening (v4.20.1):
+ *  - Lazy require of better-sqlite3 — missing native binary degrades to a clear
+ *    warning + skip (bot keeps running, falls back to legacy JSON path until
+ *    the user fixes their install).
+ *  - Pre-flight disk-space check: refuses to start if free space < 2× source.
+ *  - Progress logging every 1 000 entries on large indexes.
+ *  - Corrupt source JSON is renamed to `.broken.<timestamp>` so the next run
+ *    doesn't loop on the same parse error.
+ *
+ * Safety:
+ *  - Source JSON is renamed to .embeddings.json.bak-pre-sqlite (kept on disk).
+ *  - Entry counts are compared after import; mismatch → throw, leaving the
+ *    half-written DB removed and the source JSON untouched.
+ */
+import fs from "fs";
+import path from "path";
+import { createRequire } from "module";
+import { EMBEDDINGS_IDX, EMBEDDINGS_DB } from "../paths.js";
+const cjsRequire = createRequire(import.meta.url);
+function vectorToBlob(v) {
+    const f32 = new Float32Array(v);
+    return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
+}
+export function shouldMigrateEmbeddingsToSqlite() {
+    return fs.existsSync(EMBEDDINGS_IDX) && !fs.existsSync(EMBEDDINGS_DB);
+}
+/**
+ * Best-effort free-space probe. Returns Infinity if the platform has no
+ * statfs (which means we'll proceed without the safety check rather than
+ * blocking the migration). Node 18.15+ ships statfsSync on all major platforms.
+ */
+function freeBytesOnVolume(forPath) {
+    try {
+        const fsAny = fs;
+        if (typeof fsAny.statfsSync !== "function")
+            return Number.POSITIVE_INFINITY;
+        const stat = fsAny.statfsSync(forPath);
+        const bavail = typeof stat.bavail === "bigint" ? Number(stat.bavail) : stat.bavail;
+        const bsize = typeof stat.bsize === "bigint" ? Number(stat.bsize) : stat.bsize;
+        return bavail * bsize;
+    }
+    catch {
+        return Number.POSITIVE_INFINITY;
+    }
+}
+/**
+ * Run the migration. Returns the entry count migrated, or null if skipped.
+ */
+export function migrateEmbeddingsToSqlite() {
+    if (!shouldMigrateEmbeddingsToSqlite())
+        return null;
+    // ── Pre-flight: better-sqlite3 loadable? ───────────────────────────────────
+    let Database;
+    try {
+        Database = cjsRequire("better-sqlite3");
+    }
+    catch (err) {
+        console.warn("⚠️ Embeddings migration skipped: better-sqlite3 native binary unavailable. " +
+            "Bot continues with legacy JSON index. Fix: `npm rebuild better-sqlite3` " +
+            "or reinstall alvin-bot. Underlying error:", err instanceof Error ? err.message : err);
+        return null;
+    }
+    const sourceSize = fs.statSync(EMBEDDINGS_IDX).size;
+    // ── Pre-flight: enough free space? ─────────────────────────────────────────
+    const targetDir = path.dirname(EMBEDDINGS_DB);
+    fs.mkdirSync(targetDir, { recursive: true });
+    const free = freeBytesOnVolume(targetDir);
+    // We need source + about half of source for the SQLite file, plus headroom
+    // for WAL during the transaction. Demand 2× source size to be comfortable.
+    const required = sourceSize * 2;
+    if (free < required) {
+        console.warn(`⚠️ Embeddings migration skipped: insufficient free disk space on ${targetDir}. ` +
+            `Need ~${(required / 1024 / 1024).toFixed(0)} MB, have ${(free / 1024 / 1024).toFixed(0)} MB. ` +
+            `Free up some space and restart the bot to retry.`);
+        return null;
+    }
+    // ── Read & parse source ────────────────────────────────────────────────────
+    const t0 = Date.now();
+    console.log(`📦 Migrating embeddings JSON (${(sourceSize / 1024 / 1024).toFixed(0)} MB) → SQLite...`);
+    const raw = fs.readFileSync(EMBEDDINGS_IDX, "utf-8");
+    let legacy;
+    try {
+        legacy = JSON.parse(raw);
+    }
+    catch (err) {
+        // Move the broken JSON aside so we don't try to migrate it again next boot.
+        const broken = `${EMBEDDINGS_IDX}.broken.${Date.now()}`;
+        try {
+            fs.renameSync(EMBEDDINGS_IDX, broken);
+            console.error(`❌ Embeddings migration: source JSON is corrupt — renamed to ${path.basename(broken)} ` +
+                `and skipped. The bot will rebuild the index from scratch on first search ` +
+                `(this may incur Google API calls). Underlying parse error:`, err);
+        }
+        catch (renameErr) {
+            console.error("❌ Embeddings migration: source JSON is corrupt AND could not be renamed:", err, "Rename error:", renameErr);
+        }
+        return null;
+    }
+    const validEntries = (legacy.entries ?? []).filter(e => Array.isArray(e.vector) && e.vector.length > 0);
+    // ── Write DB ───────────────────────────────────────────────────────────────
+    const db = new Database(EMBEDDINGS_DB);
+    try {
+        db.pragma("journal_mode = WAL");
+        db.pragma("synchronous = NORMAL");
+        db.exec(`
+      CREATE TABLE IF NOT EXISTS meta (
+        key   TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS file_mtimes (
+        source   TEXT PRIMARY KEY,
+        mtime_ms REAL NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS entries (
+        id         TEXT PRIMARY KEY,
+        source     TEXT NOT NULL,
+        text       TEXT NOT NULL,
+        vector     BLOB NOT NULL,
+        indexed_at INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_entries_source ON entries(source);
+    `);
+        const setMeta = db.prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value");
+        setMeta.run("model", legacy.model || "gemini-embedding-001");
+        setMeta.run("schemaVersion", "1");
+        setMeta.run("lastReindex", String(legacy.lastReindex || 0));
+        setMeta.run("migratedFromJson", String(Date.now()));
+        const insMtime = db.prepare("INSERT INTO file_mtimes (source, mtime_ms) VALUES (?, ?) ON CONFLICT(source) DO UPDATE SET mtime_ms = excluded.mtime_ms");
+        const writeMtimes = db.transaction((rows) => {
+            for (const [s, m] of rows)
+                insMtime.run(s, m);
+        });
+        writeMtimes(Object.entries(legacy.fileMtimes ?? {}));
+        const insEntry = db.prepare("INSERT INTO entries (id, source, text, vector, indexed_at) VALUES (?, ?, ?, ?, ?)");
+        // Write entries in chunks of 1 000 so we can log progress on huge indexes.
+        const CHUNK = 1000;
+        const total = validEntries.length;
+        let written = 0;
+        const writeChunk = db.transaction((rows) => {
+            for (const e of rows) {
+                insEntry.run(e.id, e.source, e.text, vectorToBlob(e.vector), e.indexedAt);
+            }
+        });
+        for (let i = 0; i < total; i += CHUNK) {
+            const slice = validEntries.slice(i, i + CHUNK);
+            writeChunk(slice);
+            written += slice.length;
+            if (total > 5000 && (written === total || written % 5000 === 0)) {
+                console.log(`   …migrated ${written} / ${total} entries (${Math.round((written / total) * 100)} %)`);
+            }
+        }
+        const writtenCount = db.prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+        if (writtenCount !== validEntries.length) {
+            throw new Error(`Entry-count mismatch after migration: expected ${validEntries.length}, got ${writtenCount}`);
+        }
+        db.close();
+        // ── Move source JSON aside so we never re-migrate ────────────────────────
+        const bak = `${EMBEDDINGS_IDX}.bak-pre-sqlite`;
+        try {
+            fs.renameSync(EMBEDDINGS_IDX, bak);
+        }
+        catch (err) {
+            console.warn("⚠️ Could not rename source JSON (migration still succeeded):", err);
+        }
+        const targetSize = fs.statSync(EMBEDDINGS_DB).size;
+        const dt = Date.now() - t0;
+        console.log(`✅ Embeddings migrated: ${writtenCount} entries, ${(sourceSize / 1024 / 1024).toFixed(0)} MB JSON → ${(targetSize / 1024 / 1024).toFixed(0)} MB SQLite in ${dt} ms`);
+        return { entries: writtenCount, sourceMb: sourceSize / 1024 / 1024, targetMb: targetSize / 1024 / 1024 };
+    }
+    catch (err) {
+        db.close();
+        // Remove half-written DB so the next boot retries cleanly with the original JSON intact.
+        try {
+            fs.unlinkSync(EMBEDDINGS_DB);
+            // also unlink WAL/SHM if present
+            for (const ext of ["-wal", "-shm"]) {
+                const p = `${EMBEDDINGS_DB}${ext}`;
+                if (fs.existsSync(p))
+                    fs.unlinkSync(p);
+            }
+        }
+        catch {
+            /* nothing to clean */
+        }
+        throw err;
+    }
+}

package/dist/services/embeddings.js

@@ -1,31 +1,159 @@
 /**
  * Embeddings Service — Vector-based semantic memory search.
  *
- * Uses Google's text-embedding-004 model for generating embeddings.
- * Stores embeddings in a local JSON index file for fast cosine similarity search.
+ * Uses Google's gemini-embedding-001 model for generating embeddings.
+ * Stores embeddings in a SQLite database (.embeddings.db) — replaces the
+ * older .embeddings.json index since v4.20. The migration runs once
+ * automatically on startup (see src/migrate.ts).
  *
  * Architecture:
- * - Each memory entry (paragraph/section) gets an embedding vector
- * - Vectors are stored in docs/memory/.embeddings.json
- * - On query, the search text is embedded and compared via cosine similarity
- * - Top-K results returned with similarity scores
+ * - Each memory entry (paragraph/section) gets a 3072-dim Float32 vector.
+ * - Vectors are stored as raw BLOB (4 bytes × 3072 = 12 KB each) instead of
+ *   JSON-encoded Float64 arrays (~24 KB each) — halves disk footprint.
+ * - Cosine similarity runs in-memory: SQLite has no native vector ops, but
+ *   reading the BLOBs is mmap-cheap and JS does the dot product fast enough
+ *   for the current corpus (a few thousand entries).
+ * - Reindexing is per-chunk INSERT/UPDATE — no full-file rewrite.
  */
 import fs from "fs";
 import path from "path";
 import { resolve } from "path";
-import { config } from "../config.js";
 import os from "os";
-import { MEMORY_DIR, MEMORY_FILE, EMBEDDINGS_IDX as INDEX_FILE } from "../paths.js";
+import { createRequire } from "module";
+import { config } from "../config.js";
+import { MEMORY_DIR, MEMORY_FILE, EMBEDDINGS_DB } from "../paths.js";
 import { ASSETS_DIR, ASSETS_INDEX_MD } from "../paths.js";
+let SqliteClass = null;
+let sqliteLoadAttempted = false;
+let sqliteLoadError = null;
+const cjsRequire = createRequire(import.meta.url);
+function loadSqlite() {
+    if (sqliteLoadAttempted)
+        return SqliteClass;
+    sqliteLoadAttempted = true;
+    try {
+        SqliteClass = cjsRequire("better-sqlite3");
+        return SqliteClass;
+    }
+    catch (err) {
+        sqliteLoadError = err instanceof Error ? err : new Error(String(err));
+        console.warn("⚠️ better-sqlite3 native binary unavailable — embeddings disabled. " +
+            "Bot continues without semantic memory search. Fix: rebuild deps with " +
+            "`cd $(npm root -g)/alvin-bot && npm rebuild better-sqlite3` or reinstall " +
+            "alvin-bot. Underlying error: " +
+            sqliteLoadError.message);
+        return null;
+    }
+}
+export function getEmbeddingsBackendStatus() {
+    loadSqlite();
+    return { available: SqliteClass !== null, error: sqliteLoadError?.message ?? null };
+}
 // Hub memory directory (Claude Hub — read-only, additional context)
 const HUB_MEMORY_DIR = resolve(os.homedir(), ".claude", "hub", "MEMORY");
-// ── Google Embeddings API ───────────────────────────────
+// ── Constants ───────────────────────────────────────────
 const EMBEDDING_MODEL = "gemini-embedding-001";
 const EMBEDDING_DIMENSION = 3072;
+const SCHEMA_VERSION = "1";
+// ── Vector encoding (Float32Array ↔ Buffer) ─────────────
+function vectorToBlob(v) {
+    const f32 = new Float32Array(v);
+    // Buffer.from(arrayBuffer, byteOffset, length) preserves the underlying memory.
+    return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
+}
+function blobToVector(b) {
+    // Buffers from better-sqlite3 own their memory and may not be aligned to 4 bytes.
+    // Copying into a fresh Float32Array guarantees alignment.
+    const f32 = new Float32Array(b.byteLength / 4);
+    const dv = new DataView(b.buffer, b.byteOffset, b.byteLength);
+    for (let i = 0; i < f32.length; i++) {
+        f32[i] = dv.getFloat32(i * 4, true /* little-endian */);
+    }
+    return f32;
+}
+// ── DB lifecycle ────────────────────────────────────────
+let dbInstance = null;
 /**
- * Get embeddings for one or more texts via Google's API.
- * Batches up to 100 texts per request.
+ * Returns the live DB handle, or null when better-sqlite3 isn't loadable.
+ * Callers must handle the null case (treat as "search unavailable").
  */
+function db() {
+    if (dbInstance)
+        return dbInstance;
+    const Database = loadSqlite();
+    if (!Database)
+        return null;
+    // Ensure directory exists (handles fresh installs).
+    fs.mkdirSync(path.dirname(EMBEDDINGS_DB), { recursive: true });
+    dbInstance = new Database(EMBEDDINGS_DB);
+    dbInstance.pragma("journal_mode = WAL");
+    dbInstance.pragma("synchronous = NORMAL");
+    dbInstance.pragma("temp_store = MEMORY");
+    dbInstance.pragma("mmap_size = 268435456"); // 256 MB
+    dbInstance.exec(`
+    CREATE TABLE IF NOT EXISTS meta (
+      key   TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    );
+    CREATE TABLE IF NOT EXISTS file_mtimes (
+      source   TEXT PRIMARY KEY,
+      mtime_ms REAL NOT NULL
+    );
+    CREATE TABLE IF NOT EXISTS entries (
+      id         TEXT PRIMARY KEY,
+      source     TEXT NOT NULL,
+      text       TEXT NOT NULL,
+      vector     BLOB NOT NULL,
+      indexed_at INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_entries_source ON entries(source);
+  `);
+    // Initialise meta if absent.
+    const set = dbInstance.prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO NOTHING");
+    set.run("model", EMBEDDING_MODEL);
+    set.run("schemaVersion", SCHEMA_VERSION);
+    return dbInstance;
+}
+/** Close handle (used by tests / shutdown). */
+export function closeEmbeddingsDb() {
+    if (dbInstance) {
+        dbInstance.close();
+        dbInstance = null;
+    }
+}
+/** Sharper assertion for use inside helpers that require an open DB. */
+function dbOrThrow() {
+    const d = db();
+    if (!d) {
+        throw new Error("Embeddings DB unavailable — better-sqlite3 native module not loaded");
+    }
+    return d;
+}
+// ── Meta helpers ────────────────────────────────────────
+function getMeta(key) {
+    const row = dbOrThrow().prepare("SELECT value FROM meta WHERE key = ?").get(key);
+    return row?.value ?? null;
+}
+function setMeta(key, value) {
+    dbOrThrow()
+        .prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value")
+        .run(key, value);
+}
+function getFileMtimes() {
+    const rows = dbOrThrow()
+        .prepare("SELECT source, mtime_ms FROM file_mtimes")
+        .all();
+    const out = {};
+    for (const r of rows)
+        out[r.source] = r.mtime_ms;
+    return out;
+}
+function setFileMtime(source, mtimeMs) {
+    dbOrThrow()
+        .prepare("INSERT INTO file_mtimes (source, mtime_ms) VALUES (?, ?) ON CONFLICT(source) DO UPDATE SET mtime_ms = excluded.mtime_ms")
+        .run(source, mtimeMs);
+}
+// ── Google Embeddings API ───────────────────────────────
 async function getEmbeddings(texts) {
     const apiKey = config.apiKeys.google;
     if (!apiKey) {
@@ -50,16 +178,13 @@ async function getEmbeddings(texts) {
             const err = await response.text();
             throw new Error(`Embedding API error: ${response.status} — ${err}`);
         }
-        const data = await response.json();
+        const data = (await response.json());
         for (const emb of data.embeddings) {
             results.push(emb.values);
         }
     }
     return results;
 }
-/**
- * Get embedding for a single query text.
- */
 async function getQueryEmbedding(text) {
     const apiKey = config.apiKeys.google;
     if (!apiKey) {
@@ -78,11 +203,11 @@ async function getQueryEmbedding(text) {
         const err = await response.text();
         throw new Error(`Embedding API error: ${response.status} — ${err}`);
     }
-    const data = await response.json();
+    const data = (await response.json());
     return data.embedding.values;
 }
 // ── Vector Math ─────────────────────────────────────────
-function cosineSimilarity(a, b) {
+function cosineSimilarityF32(a, b) {
     if (a.length !== b.length)
         return 0;
     let dotProduct = 0;
@@ -97,20 +222,13 @@ function cosineSimilarity(a, b) {
     return denom === 0 ? 0 : dotProduct / denom;
 }
 // ── Text Chunking ───────────────────────────────────────
-/**
- * Split a markdown file into meaningful chunks.
- * Splits on ## headers, keeping each section as a chunk.
- * Falls back to paragraph splitting for files without headers.
- */
 function chunkMarkdown(content, source) {
     const chunks = [];
-    // Split on ## headers
     const sections = content.split(/^(?=## )/gm);
     for (let i = 0; i < sections.length; i++) {
         const section = sections[i].trim();
         if (!section || section.length < 20)
-            continue; // Skip tiny sections
-        // If section is too long (>1000 chars), split into paragraphs
+            continue;
         if (section.length > 1000) {
             const paragraphs = section.split(/\n\n+/);
             let currentChunk = "";
@@ -142,51 +260,7 @@ function chunkMarkdown(content, source) {
     }
     return chunks;
 }
-// ── Index Management ────────────────────────────────────
-// In-memory cache for the embedding index. Without this, every query would
-// re-read and re-parse the on-disk index (can be 100+ MB, making searchMemory
-// the slowest step in a message turn). We keep the parsed object and invalidate
-// via mtime check — so external reindexers are still picked up.
-let indexCache = null;
-let indexCacheMtime = 0;
-function loadIndex() {
-    try {
-        const st = fs.statSync(INDEX_FILE);
-        if (indexCache && st.mtimeMs === indexCacheMtime) {
-            return indexCache;
-        }
-        const raw = fs.readFileSync(INDEX_FILE, "utf-8");
-        indexCache = JSON.parse(raw);
-        indexCacheMtime = st.mtimeMs;
-        return indexCache;
-    }
-    catch {
-        // File missing or unparseable — return an empty index and don't cache it
-        // (next call will retry, so a freshly-written index gets picked up).
-        return {
-            model: EMBEDDING_MODEL,
-            lastReindex: 0,
-            fileMtimes: {},
-            entries: [],
-        };
-    }
-}
-function saveIndex(index) {
-    fs.writeFileSync(INDEX_FILE, JSON.stringify(index));
-    // Refresh cache immediately so the next loadIndex() sees the new state
-    // without a disk round-trip.
-    indexCache = index;
-    try {
-        indexCacheMtime = fs.statSync(INDEX_FILE).mtimeMs;
-    }
-    catch {
-        indexCacheMtime = Date.now();
-    }
-}
-/**
- * Recursively walk a directory, returning file paths.
- * Skips INDEX.json and INDEX.md at the directory root.
- */
+// ── Indexable file discovery ────────────────────────────
 function walkAssetDir(dir) {
     const results = [];
     function walk(currentDir) {
@@ -213,17 +287,11 @@ function walkAssetDir(dir) {
     return results;
 }
 const TEXT_EXTENSIONS = new Set([".md", ".html", ".txt", ".css", ".ts"]);
-/**
- * Get all files that should be indexed — memories + text-based assets.
- */
 function getIndexableFiles() {
     const files = [];
-    // ── Memories (existing) ───────────────────────────────
-    // Alvin-Bot MEMORY.md
     if (fs.existsSync(MEMORY_FILE)) {
         files.push({ path: MEMORY_FILE, relativePath: "MEMORY.md" });
     }
-    // Alvin-Bot daily logs
     if (fs.existsSync(MEMORY_DIR)) {
         const entries = fs.readdirSync(MEMORY_DIR);
         for (const entry of entries) {
@@ -235,7 +303,6 @@ function getIndexableFiles() {
             }
         }
     }
-    // Hub memories (~/.claude/hub/MEMORY/) — Claude Hub knowledge base
     if (fs.existsSync(HUB_MEMORY_DIR)) {
         try {
             const entries = fs.readdirSync(HUB_MEMORY_DIR);
@@ -248,14 +315,13 @@ function getIndexableFiles() {
                 }
             }
         }
-        catch { /* Hub not available — skip */ }
+        catch {
+            /* Hub not available — skip */
+        }
     }
-    // ── Assets (new) ──────────────────────────────────────
-    // Asset INDEX.md — compact summary of all assets
     if (fs.existsSync(ASSETS_INDEX_MD)) {
         files.push({ path: ASSETS_INDEX_MD, relativePath: "assets/INDEX.md" });
     }
-    // Text-based asset files (HTML, MD, TXT, CSS, TS)
     if (fs.existsSync(ASSETS_DIR)) {
         for (const entry of walkAssetDir(ASSETS_DIR)) {
             if (TEXT_EXTENSIONS.has(path.extname(entry.name))) {
@@ -268,120 +334,142 @@ function getIndexableFiles() {
     }
     return files;
 }
-/**
- * Check which files need reindexing (new or modified).
- */
-function getStaleFiles(index) {
+function getStaleFiles() {
     const allFiles = getIndexableFiles();
+    const known = getFileMtimes();
     const stale = [];
     for (const file of allFiles) {
         try {
-            const stat = fs.statSync(file.path);
-            const mtime = stat.mtimeMs;
-            if (!index.fileMtimes[file.relativePath] || index.fileMtimes[file.relativePath] < mtime) {
+            const mtime = fs.statSync(file.path).mtimeMs;
+            if (!known[file.relativePath] || known[file.relativePath] < mtime) {
                 stale.push(file);
             }
         }
         catch {
-            // File disappeared — skip
+            /* file disappeared */
         }
     }
     return stale;
 }
 // ── Public API ──────────────────────────────────────────
-/**
- * Reindex all memory files (or just stale ones).
- * Returns number of chunks indexed.
- */
 export async function reindexMemory(force = false) {
-    const index = loadIndex();
-    const filesToIndex = force ? getIndexableFiles() : getStaleFiles(index);
+    if (!loadSqlite()) {
+        return { indexed: 0, total: 0 };
+    }
+    const filesToIndex = force ? getIndexableFiles() : getStaleFiles();
     if (filesToIndex.length === 0) {
-        return { indexed: 0, total: index.entries.length };
+        const total = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+        return { indexed: 0, total };
     }
-    // Remove old entries for files being reindexed
-    const reindexSources = new Set(filesToIndex.map(f => f.relativePath));
-    index.entries = index.entries.filter(e => !reindexSources.has(e.source));
-    // Chunk all files
+    // Drop existing entries for files being reindexed (per-source DELETE is O(log n) thanks to idx).
+    const delStmt = dbOrThrow().prepare("DELETE FROM entries WHERE source = ?");
+    const dropOld = dbOrThrow().transaction((sources) => {
+        for (const s of sources)
+            delStmt.run(s);
+    });
+    dropOld(filesToIndex.map(f => f.relativePath));
+    // Chunk all files.
     const allChunks = [];
     for (const file of filesToIndex) {
         try {
             const content = fs.readFileSync(file.path, "utf-8");
             const chunks = chunkMarkdown(content, file.relativePath);
+            const mtime = fs.statSync(file.path).mtimeMs;
             for (const chunk of chunks) {
-                allChunks.push({ ...chunk, source: file.relativePath });
+                allChunks.push({ ...chunk, source: file.relativePath, mtime });
             }
-            // Update mtime
-            const stat = fs.statSync(file.path);
-            index.fileMtimes[file.relativePath] = stat.mtimeMs;
         }
         catch (err) {
             console.error(`Failed to chunk ${file.relativePath}:`, err);
         }
     }
     if (allChunks.length === 0) {
-        saveIndex(index);
-        return { indexed: 0, total: index.entries.length };
+        // Even with zero chunks, keep mtimes in sync so we don't re-walk on next run.
+        const updMtime = dbOrThrow().transaction((files) => {
+            for (const f of files) {
+                try {
+                    setFileMtime(f.relativePath, fs.statSync(f.path).mtimeMs);
+                }
+                catch {
+                    /* file disappeared */
+                }
+            }
+        });
+        updMtime(filesToIndex);
+        const total = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+        return { indexed: 0, total };
     }
-    // Get embeddings for all chunks
+    // Get embeddings for all chunks (network).
     const texts = allChunks.map(c => c.text);
     const vectors = await getEmbeddings(texts);
-    // Add to index
-    for (let i = 0; i < allChunks.length; i++) {
-        index.entries.push({
-            id: allChunks[i].id,
-            source: allChunks[i].source,
-            text: allChunks[i].text,
-            vector: vectors[i],
-            indexedAt: Date.now(),
-        });
-    }
-    index.lastReindex = Date.now();
-    saveIndex(index);
-    return { indexed: allChunks.length, total: index.entries.length };
+    // Single transaction for all writes.
+    const insertStmt = dbOrThrow().prepare("INSERT INTO entries (id, source, text, vector, indexed_at) VALUES (?, ?, ?, ?, ?) " +
+        "ON CONFLICT(id) DO UPDATE SET source=excluded.source, text=excluded.text, vector=excluded.vector, indexed_at=excluded.indexed_at");
+    const writeAll = dbOrThrow().transaction((rows) => {
+        for (const r of rows) {
+            insertStmt.run(r.id, r.source, r.text, r.vector, r.indexedAt);
+        }
+    });
+    const now = Date.now();
+    writeAll(allChunks.map((c, i) => ({
+        id: c.id,
+        source: c.source,
+        text: c.text,
+        vector: vectorToBlob(vectors[i]),
+        indexedAt: now,
+    })));
+    // Update mtimes for the files we just (re-)indexed.
+    const updMtime = dbOrThrow().transaction((files) => {
+        for (const f of files) {
+            try {
+                setFileMtime(f.relativePath, fs.statSync(f.path).mtimeMs);
+            }
+            catch {
+                /* file disappeared */
+            }
+        }
+    });
+    updMtime(filesToIndex);
+    setMeta("lastReindex", String(now));
+    const total = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+    return { indexed: allChunks.length, total };
 }
-/**
- * Semantic search across all indexed memory.
- * Returns top-K results sorted by similarity.
- */
 export async function searchMemory(query, topK = 5, minScore = 0.3) {
-    const index = loadIndex();
-    if (index.entries.length === 0) {
-        // Auto-index if empty
+    if (!loadSqlite()) {
+        return [];
+    }
+    // Auto-index if empty.
+    const total = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+    if (total === 0) {
         await reindexMemory();
-        // Reload
-        const reloaded = loadIndex();
-        if (reloaded.entries.length === 0)
+        const after = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+        if (after === 0)
             return [];
     }
-    // Get query embedding
-    const queryVector = await getQueryEmbedding(query);
-    // Calculate similarities
-    const scored = index.entries.map(entry => ({
-        text: entry.text,
-        source: entry.source,
-        score: cosineSimilarity(queryVector, entry.vector),
-    }));
-    // Sort by score descending, filter by minScore, take topK
-    return scored
-        .filter(r => r.score >= minScore)
-        .sort((a, b) => b.score - a.score)
-        .slice(0, topK);
+    const queryVector = Float32Array.from(await getQueryEmbedding(query));
+    const rows = dbOrThrow().prepare("SELECT id, source, text, vector FROM entries").all();
+    const scored = [];
+    for (const row of rows) {
+        const v = blobToVector(row.vector);
+        const score = cosineSimilarityF32(queryVector, v);
+        if (score >= minScore) {
+            scored.push({ text: row.text, source: row.source, score });
+        }
+    }
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, topK);
 }
-/**
- * Get index stats for /status.
- */
-/**
- * Auto-reindex on startup. Indexes only stale/new files (incremental).
- * Runs in background — does not block bot startup.
- */
 export async function initEmbeddings() {
+    if (!loadSqlite()) {
+        return; // already warned via loadSqlite
+    }
     try {
-        const stale = getStaleFiles(loadIndex());
+        db(); // Open & migrate schema.
+        const stale = getStaleFiles();
         if (stale.length === 0) {
-            const idx = loadIndex();
-            if (idx.entries.length > 0)
-                return; // Already indexed, nothing stale
+            const total = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+            if (total > 0)
+                return;
         }
         const result = await reindexMemory();
         if (result.indexed > 0) {
@@ -389,21 +477,29 @@ export async function initEmbeddings() {
         }
     }
     catch (err) {
-        // Non-fatal — bot works without embeddings
         console.warn("⚠️ Embeddings init failed:", err instanceof Error ? err.message : err);
     }
 }
 export function getIndexStats() {
-    const index = loadIndex();
+    let entries = 0;
+    let files = 0;
+    let lastReindex = 0;
     let sizeBytes = 0;
+    if (!loadSqlite()) {
+        return { entries, files, lastReindex, sizeBytes };
+    }
     try {
-        sizeBytes = fs.statSync(INDEX_FILE).size;
+        entries = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
+        files = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM file_mtimes").get().c;
+        const meta = getMeta("lastReindex");
+        if (meta)
+            lastReindex = Number(meta);
+        sizeBytes = fs.statSync(EMBEDDINGS_DB).size;
+    }
+    catch {
+        /* DB not yet initialised */
     }
-    catch { /* empty */ }
-    return {
-        entries: index.entries.length,
-        files: Object.keys(index.fileMtimes).length,
-        lastReindex: index.lastReindex,
-        sizeBytes,
-    };
+    return { entries, files, lastReindex, sizeBytes };
 }
+// ── Re-export embedding dim for tests / debugging ──────
+export { EMBEDDING_DIMENSION, EMBEDDING_MODEL };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "alvin-bot",
-  "version": "4.19.2",
-  "description": "Alvin Bot \u2014 Your personal AI agent on Telegram, WhatsApp, Discord, Signal, and Web.",
+  "version": "4.20.1",
+  "description": "Alvin Bot — Your personal AI agent on Telegram, WhatsApp, Discord, Signal, and Web.",
   "type": "module",
   "main": "dist/index.js",
   "bin": {
@@ -170,6 +170,7 @@
     "@types/node": "^22.0.0",
     "@types/ws": "^8.18.1",
     "@whiskeysockets/baileys": "^6.7.21",
+    "better-sqlite3": "^12.9.0",
     "dotenv": "^16.4.0",
     "electron-updater": "^6.8.3",
     "grammy": "^1.30.0",
@@ -181,6 +182,7 @@
     "ws": "^8.19.0"
   },
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
     "@vitest/ui": "^4.1.4",
     "electron": "^35.7.5",
     "electron-builder": "^26.8.1",