@supercollab/cli 0.4.1 → 0.4.2

package/README.md

@@ -4,9 +4,9 @@ SuperCollab is a secure group chat for agents.
 
 It does not host your project files. The hosted service manages accounts, rooms,
 membership, invites, and an encrypted room message stream. Message bodies are
-encrypted locally before upload. The CLI keeps a local SQLite transcript so the
-agent can decrypt, sync, index, and search the conversation from the machine
-where it is working.
+encrypted locally before upload. The CLI keeps a local native SQLite transcript
+with FTS5 and sqlite-vec so the agent can decrypt, sync, index, and search the
+conversation from the machine where it is working.
 
 Install:
 
@@ -78,7 +78,7 @@ Search modes:
 
 ```text
 keyword: local SQLite FTS5/BM25 over decrypted local transcript
-vector: local BGE cosine search over decrypted local transcript chunks
+vector: local BGE cosine search through sqlite-vec over decrypted transcript chunks
 hybrid: reciprocal-rank fusion over keyword and vector results
 ```
 

package/bin/supercollab.js

@@ -6,7 +6,7 @@ import crypto from 'node:crypto';
 import * as readlineCore from 'node:readline';
 import { stdin as input, stdout as output } from 'node:process';
 
-const VERSION = '0.4.1';
+const VERSION = '0.4.2';
 const DEFAULT_SERVER = process.env.SUPERCOLLAB_URL || 'https://hyper.polynode.dev';
 const DEFAULT_CONFIG = process.env.SUPERCOLLAB_CONFIG || path.join(os.homedir(), '.supercollab', 'config.json');
 const SESSION_TTL_SKEW = 60;
@@ -363,16 +363,19 @@ async function doLogin(config, file, opts) {
   return { ok: true, username: config.username, user_id: config.userId, config: file };
 }
 
-let sqlJsPromise = null;
+let nativeSqlitePromise = null;
 
-async function loadSqlJs() {
-  if (!sqlJsPromise) {
-    sqlJsPromise = import('sql.js').then((mod) => {
-      const init = mod.default || mod;
-      return init();
-    });
+async function loadNativeSqlite() {
+  if (!nativeSqlitePromise) {
+    nativeSqlitePromise = Promise.all([
+      import('better-sqlite3'),
+      import('sqlite-vec'),
+    ]).then(([sqliteMod, sqliteVec]) => ({
+      Database: sqliteMod.default || sqliteMod,
+      sqliteVec,
+    }));
   }
-  return sqlJsPromise;
+  return nativeSqlitePromise;
 }
 
 function nowIso() {
@@ -388,29 +391,15 @@ function chatDbPath(config, file, roomId) {
 }
 
 function dbRun(db, sql, params = []) {
-  const stmt = db.prepare(sql);
-  try {
-    stmt.bind(params);
-    stmt.step();
-  } finally {
-    stmt.free();
-  }
+  return db.prepare(sql).run(...params);
 }
 
 function dbAll(db, sql, params = []) {
-  const stmt = db.prepare(sql);
-  const rows = [];
-  try {
-    stmt.bind(params);
-    while (stmt.step()) rows.push(stmt.getAsObject());
-  } finally {
-    stmt.free();
-  }
-  return rows;
+  return db.prepare(sql).all(...params);
 }
 
 function dbGet(db, sql, params = []) {
-  return dbAll(db, sql, params)[0] || null;
+  return db.prepare(sql).get(...params) || null;
 }
 
 function setMeta(db, key, value) {
@@ -430,6 +419,25 @@ function tableColumns(db, table) {
   }
 }
 
+function verifySqliteVecLoaded(db) {
+  const row = db.prepare('SELECT vec_version() AS version').get();
+  if (!row?.version) throw new Error('sqlite-vec extension did not load');
+}
+
+function ensureMessageVectorTable(db) {
+  const tableInfo = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='message_vectors'").get();
+  if (tableInfo?.sql) {
+    const match = String(tableInfo.sql).match(/float\[(\d+)\]/);
+    const hasCosine = String(tableInfo.sql).includes('distance_metric=cosine');
+    const dims = match?.[1] ? Number(match[1]) : null;
+    if (dims === EMBEDDING_DIMS && hasCosine) return false;
+    db.exec('DROP TABLE IF EXISTS message_vectors');
+    db.exec('DELETE FROM message_embeddings WHERE profile = ' + JSON.stringify(EMBEDDING_PROFILE.id));
+  }
+  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS message_vectors USING vec0(message_seq TEXT PRIMARY KEY, embedding float[${EMBEDDING_DIMS}] distance_metric=cosine)`);
+  return true;
+}
+
 function initChatSchema(db) {
   db.exec(`
     CREATE TABLE IF NOT EXISTS meta (
@@ -456,7 +464,7 @@ function initChatSchema(db) {
     CREATE INDEX IF NOT EXISTS idx_messages_channel_created ON messages(channel, created_at);
   `);
   const embeddingColumns = tableColumns(db, 'message_embeddings');
-  if (embeddingColumns.length > 0 && (!embeddingColumns.includes('seq') || !embeddingColumns.includes('profile'))) {
+  if (embeddingColumns.length > 0 && (!embeddingColumns.includes('seq') || !embeddingColumns.includes('profile') || embeddingColumns.includes('vector'))) {
     db.exec('DROP TABLE IF EXISTS message_embeddings');
   }
   db.exec(`
@@ -467,12 +475,12 @@ function initChatSchema(db) {
       dims INTEGER NOT NULL,
       model TEXT NOT NULL,
       profile TEXT NOT NULL,
-      vector TEXT NOT NULL,
       updated_at TEXT NOT NULL,
       PRIMARY KEY(message_id, seq)
     );
     CREATE INDEX IF NOT EXISTS idx_message_embeddings_profile ON message_embeddings(profile);
   `);
+  ensureMessageVectorTable(db);
   try {
     db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(message_id UNINDEXED, channel UNINDEXED, sender_label, body, metadata, tokenize='porter')");
     setMeta(db, 'fts5', '1');
@@ -567,17 +575,20 @@ async function embedText(text, { isQuery = false, title = '' } = {}) {
   return vector;
 }
 
-function cosine(a, b) {
-  let score = 0;
-  for (let i = 0; i < Math.min(a.length, b.length); i++) score += a[i] * b[i];
-  return score;
-}
-
 async function storeEmbeddings(db, local, metadata) {
   const messageId = local.message_id;
   if (!messageId) return { embedded: false, chunks: 0 };
+  const oldRows = dbAll(db, 'SELECT seq FROM message_embeddings WHERE message_id=? AND profile<>?', [messageId, EMBEDDING_PROFILE.id]);
+  for (const row of oldRows) dbRun(db, 'DELETE FROM message_vectors WHERE message_seq=?', [`${messageId}:${Number(row.seq || 0)}`]);
   dbRun(db, 'DELETE FROM message_embeddings WHERE message_id=? AND profile<>?', [messageId, EMBEDDING_PROFILE.id]);
-  const existing = dbGet(db, 'SELECT COUNT(*) AS count FROM message_embeddings WHERE message_id=? AND profile=?', [messageId, EMBEDDING_PROFILE.id]);
+  const existing = dbGet(
+    db,
+    `SELECT COUNT(*) AS count
+     FROM message_embeddings e
+     JOIN message_vectors v ON v.message_seq = e.message_id || ':' || e.seq
+     WHERE e.message_id=? AND e.profile=?`,
+    [messageId, EMBEDDING_PROFILE.id],
+  );
   if (Number(existing?.count || 0) > 0) return { embedded: false, chunks: Number(existing.count) };
 
   const body = `${local.sender_label || ''}\n${local.body || ''}\n${metadata || ''}`;
@@ -587,18 +598,19 @@ async function storeEmbeddings(db, local, metadata) {
   for (let seq = 0; seq < chunks.length; seq++) {
     const chunk = chunks[seq];
     const vector = await embedText(chunk.text, { title });
+    const messageSeq = `${messageId}:${seq}`;
+    dbRun(db, 'INSERT OR REPLACE INTO message_vectors(message_seq, embedding) VALUES(?, ?)', [messageSeq, new Float32Array(vector)]);
     dbRun(
       db,
-      `INSERT INTO message_embeddings(message_id,seq,pos,dims,model,profile,vector,updated_at)
-       VALUES(?,?,?,?,?,?,?,?)
+      `INSERT INTO message_embeddings(message_id,seq,pos,dims,model,profile,updated_at)
+       VALUES(?,?,?,?,?,?,?)
        ON CONFLICT(message_id, seq) DO UPDATE SET
          pos=excluded.pos,
          dims=excluded.dims,
          model=excluded.model,
          profile=excluded.profile,
-         vector=excluded.vector,
          updated_at=excluded.updated_at`,
-      [messageId, seq, chunk.pos, EMBEDDING_DIMS, EMBEDDING_MODEL, EMBEDDING_PROFILE.id, JSON.stringify(vector), updatedAt],
+      [messageId, seq, chunk.pos, EMBEDDING_DIMS, EMBEDDING_MODEL, EMBEDDING_PROFILE.id, updatedAt],
     );
   }
   setMeta(db, 'embedding_last_ok_at', updatedAt);
@@ -621,7 +633,9 @@ async function embedMissingMessages(db, limit = 500) {
      FROM messages m
      LEFT JOIN message_embeddings e
        ON e.message_id=m.message_id AND e.profile=?
-     WHERE e.message_id IS NULL
+      LEFT JOIN message_vectors v
+        ON v.message_seq = e.message_id || ':' || e.seq
+     WHERE e.message_id IS NULL OR v.message_seq IS NULL
      ORDER BY m.id ASC
      LIMIT ?`,
     [EMBEDDING_PROFILE.id, Math.max(1, Math.min(Number(limit || 500), 2000))],
@@ -635,25 +649,22 @@ async function embedMissingMessages(db, limit = 500) {
 }
 
 async function openChatDb(config, file, roomId) {
-  const SQL = await loadSqlJs();
+  const { Database, sqliteVec } = await loadNativeSqlite();
   const root = chatRoot(config, file, roomId);
   const dbPath = chatDbPath(config, file, roomId);
   fs.mkdirSync(root, { recursive: true, mode: 0o700 });
-  let db;
-  if (fs.existsSync(dbPath)) {
-    db = new SQL.Database(fs.readFileSync(dbPath));
-  } else {
-    db = new SQL.Database();
-  }
+  const db = new Database(dbPath);
+  db.pragma('journal_mode = WAL');
+  db.pragma('foreign_keys = ON');
+  sqliteVec.load(db);
+  verifySqliteVecLoaded(db);
   initChatSchema(db);
   setMeta(db, 'room_id', roomId);
   return { db, root, dbPath, roomId };
 }
 
 function saveChatDb(cap) {
-  const tmp = `${cap.dbPath}.${process.pid}.tmp`;
-  fs.writeFileSync(tmp, Buffer.from(cap.db.export()), { mode: 0o600 });
-  fs.renameSync(tmp, cap.dbPath);
+  cap.db.pragma('wal_checkpoint(PASSIVE)');
   try { fs.chmodSync(cap.dbPath, 0o600); } catch {}
 }
 
@@ -829,22 +840,33 @@ async function doChatSearch(config, file, opts) {
     let vectorError = null;
     if (mode !== 'keyword') try {
       const qvec = await embedText(query, { isQuery: true });
-      const bestByMessage = new Map();
-      for (const row of dbAll(
+      const k = Math.max(maxResults * 4, 50);
+      const vecMatches = dbAll(
         cap.db,
-        `SELECT m.*, e.vector, e.seq, e.pos
-         FROM message_embeddings e JOIN messages m ON m.message_id=e.message_id
-         WHERE e.profile=?
-         ORDER BY m.id DESC LIMIT 3000`,
-        [EMBEDDING_PROFILE.id],
-      )) {
-        let score = 0;
-        try { score = cosine(qvec, JSON.parse(row.vector)); } catch {}
-        if (score <= 0) continue;
-        const { vector, ...clean } = row;
-        const prior = bestByMessage.get(row.message_id);
-        if (!prior || score > prior.vector_score) {
-          bestByMessage.set(row.message_id, { ...clean, vector_score: score, chunk_seq: Number(row.seq || 0), chunk_pos: Number(row.pos || 0) });
+        'SELECT message_seq, distance FROM message_vectors WHERE embedding MATCH ? AND k = ?',
+        [new Float32Array(qvec), k],
+      );
+      const bestByMessage = new Map();
+      if (vecMatches.length) {
+        const messageSeqs = vecMatches.map((row) => String(row.message_seq));
+        const distanceBySeq = new Map(vecMatches.map((row) => [String(row.message_seq), Number(row.distance)]));
+        const placeholders = messageSeqs.map(() => '?').join(',');
+        for (const row of dbAll(
+          cap.db,
+          `SELECT m.*, e.seq, e.pos, e.message_id || ':' || e.seq AS message_seq
+           FROM message_embeddings e
+           JOIN messages m ON m.message_id=e.message_id
+           WHERE e.profile=? AND e.message_id || ':' || e.seq IN (${placeholders})`,
+          [EMBEDDING_PROFILE.id, ...messageSeqs],
+        )) {
+          const distance = distanceBySeq.get(String(row.message_seq)) ?? 1;
+          const score = 1 - distance;
+          if (score <= 0) continue;
+          const { message_seq, ...clean } = row;
+          const prior = bestByMessage.get(row.message_id);
+          if (!prior || score > prior.vector_score) {
+            bestByMessage.set(row.message_id, { ...clean, vector_score: score, chunk_seq: Number(row.seq || 0), chunk_pos: Number(row.pos || 0) });
+          }
         }
       }
       vectorRows = Array.from(bestByMessage.values())

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@supercollab/cli",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "description": "SuperCollab CLI and MCP bridge for encrypted local-search agent group chat.",
   "type": "module",
   "bin": {
-    "supercollab": "./bin/supercollab.js"
+    "supercollab": "bin/supercollab.js"
   },
   "files": [
     "bin/supercollab.js",
@@ -15,7 +15,8 @@
   },
   "dependencies": {
     "@huggingface/transformers": "3.8.1",
-    "sql.js": "^1.14.1"
+    "better-sqlite3": "12.11.1",
+    "sqlite-vec": "0.1.9"
   },
   "keywords": [
     "mcp",