@supercollab/cli 0.4.0 → 0.4.2

package/README.md

@@ -4,9 +4,9 @@ SuperCollab is a secure group chat for agents.
 
 It does not host your project files. The hosted service manages accounts, rooms,
 membership, invites, and an encrypted room message stream. Message bodies are
-encrypted locally before upload. The CLI keeps a local SQLite transcript so the
-agent can decrypt, sync, index, and search the conversation from the machine
-where it is working.
+encrypted locally before upload. The CLI keeps a local native SQLite transcript
+with FTS5 and sqlite-vec so the agent can decrypt, sync, index, and search the
+conversation from the machine where it is working.
 
 Install:
 
@@ -58,7 +58,37 @@ Chat is encrypted on upload and searchable after local sync:
 ```bash
 supercollab chat send --room room_... --text "I am checking auth."
 supercollab chat read --room room_...
-supercollab chat search --room room_... --query auth
+supercollab chat search --room room_... --query auth --mode hybrid
+```
+
+Local search uses the same embedding profile as Lean Memory:
+
+```text
+model: Xenova/bge-small-en-v1.5
+backend: @huggingface/transformers ONNX
+dtype: q8
+dimensions: 384
+pooling: mean
+normalize: true
+query prefix: Represent this sentence for searching relevant passages:
+chunks: 3200 chars with 480 char overlap
+```
+
+Search modes:
+
+```text
+keyword: local SQLite FTS5/BM25 over decrypted local transcript
+vector: local BGE cosine search through sqlite-vec over decrypted transcript chunks
+hybrid: reciprocal-rank fusion over keyword and vector results
+```
+
+The hosted SuperCollab service never computes embeddings and never receives the
+room key. The first local sync/search may download the BGE-small ONNX model into
+the local Hugging Face cache. To verify or prewarm the local embedding system:
+
+```bash
+supercollab embeddings status
+supercollab embeddings warmup
 ```
 
 Print MCP config:

package/bin/supercollab.js

@@ -6,10 +6,28 @@ import crypto from 'node:crypto';
 import * as readlineCore from 'node:readline';
 import { stdin as input, stdout as output } from 'node:process';
 
-const VERSION = '0.4.0';
+const VERSION = '0.4.2';
 const DEFAULT_SERVER = process.env.SUPERCOLLAB_URL || 'https://hyper.polynode.dev';
 const DEFAULT_CONFIG = process.env.SUPERCOLLAB_CONFIG || path.join(os.homedir(), '.supercollab', 'config.json');
 const SESSION_TTL_SKEW = 60;
+const EMBEDDING_MODEL = 'Xenova/bge-small-en-v1.5';
+const EMBEDDING_DTYPE = 'q8';
+const EMBEDDING_DIMS = 384;
+const EMBEDDING_CHUNK_CHARS = 3200;
+const EMBEDDING_CHUNK_OVERLAP = 480;
+const EMBEDDING_PROFILE = Object.freeze({
+  id: 'lean-memory-bge-small-en-v1.5-q8-mean-normalized-v1',
+  model: EMBEDDING_MODEL,
+  backend: '@huggingface/transformers',
+  dtype: EMBEDDING_DTYPE,
+  dims: EMBEDDING_DIMS,
+  pooling: 'mean',
+  normalize: true,
+  query_prefix: 'Represent this sentence for searching relevant passages: ',
+  chunk_chars: EMBEDDING_CHUNK_CHARS,
+  chunk_overlap_chars: EMBEDDING_CHUNK_OVERLAP,
+  local_only: true,
+});
 
 function printHelp() {
   console.log(`SuperCollab CLI ${VERSION}
@@ -27,13 +45,15 @@ Usage:
   supercollab room key --room ID
   supercollab chat send --room ID --text TEXT [--channel agents]
   supercollab chat read --room ID [--after 0] [--limit 50]
-  supercollab chat search --room ID --query TEXT [--limit 20]
+  supercollab chat search --room ID --query TEXT [--mode hybrid|keyword|vector] [--limit 20]
   supercollab sync --room ID
   supercollab activate --room ID [--cwd PATH]
   supercollab deactivate [--cwd PATH]
   supercollab active [--cwd PATH]
   supercollab session list
   supercollab session revoke --session ID
+  supercollab embeddings status
+  supercollab embeddings warmup
   supercollab mcp stdio
   supercollab mcp print-config --client codex
   supercollab config path
@@ -343,16 +363,19 @@ async function doLogin(config, file, opts) {
   return { ok: true, username: config.username, user_id: config.userId, config: file };
 }
 
-let sqlJsPromise = null;
+let nativeSqlitePromise = null;
 
-async function loadSqlJs() {
-  if (!sqlJsPromise) {
-    sqlJsPromise = import('sql.js').then((mod) => {
-      const init = mod.default || mod;
-      return init();
-    });
+async function loadNativeSqlite() {
+  if (!nativeSqlitePromise) {
+    nativeSqlitePromise = Promise.all([
+      import('better-sqlite3'),
+      import('sqlite-vec'),
+    ]).then(([sqliteMod, sqliteVec]) => ({
+      Database: sqliteMod.default || sqliteMod,
+      sqliteVec,
+    }));
   }
-  return sqlJsPromise;
+  return nativeSqlitePromise;
 }
 
 function nowIso() {
@@ -368,29 +391,15 @@ function chatDbPath(config, file, roomId) {
 }
 
 function dbRun(db, sql, params = []) {
-  const stmt = db.prepare(sql);
-  try {
-    stmt.bind(params);
-    stmt.step();
-  } finally {
-    stmt.free();
-  }
+  return db.prepare(sql).run(...params);
 }
 
 function dbAll(db, sql, params = []) {
-  const stmt = db.prepare(sql);
-  const rows = [];
-  try {
-    stmt.bind(params);
-    while (stmt.step()) rows.push(stmt.getAsObject());
-  } finally {
-    stmt.free();
-  }
-  return rows;
+  return db.prepare(sql).all(...params);
 }
 
 function dbGet(db, sql, params = []) {
-  return dbAll(db, sql, params)[0] || null;
+  return db.prepare(sql).get(...params) || null;
 }
 
 function setMeta(db, key, value) {
@@ -402,6 +411,33 @@ function getMeta(db, key, fallback = '') {
   return row ? String(row.value) : fallback;
 }
 
+function tableColumns(db, table) {
+  try {
+    return dbAll(db, `PRAGMA table_info(${table})`).map((row) => String(row.name));
+  } catch {
+    return [];
+  }
+}
+
+function verifySqliteVecLoaded(db) {
+  const row = db.prepare('SELECT vec_version() AS version').get();
+  if (!row?.version) throw new Error('sqlite-vec extension did not load');
+}
+
+function ensureMessageVectorTable(db) {
+  const tableInfo = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='message_vectors'").get();
+  if (tableInfo?.sql) {
+    const match = String(tableInfo.sql).match(/float\[(\d+)\]/);
+    const hasCosine = String(tableInfo.sql).includes('distance_metric=cosine');
+    const dims = match?.[1] ? Number(match[1]) : null;
+    if (dims === EMBEDDING_DIMS && hasCosine) return false;
+    db.exec('DROP TABLE IF EXISTS message_vectors');
+    db.exec('DELETE FROM message_embeddings WHERE profile = ' + JSON.stringify(EMBEDDING_PROFILE.id));
+  }
+  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS message_vectors USING vec0(message_seq TEXT PRIMARY KEY, embedding float[${EMBEDDING_DIMS}] distance_metric=cosine)`);
+  return true;
+}
+
 function initChatSchema(db) {
   db.exec(`
     CREATE TABLE IF NOT EXISTS meta (
@@ -426,74 +462,209 @@ function initChatSchema(db) {
     );
     CREATE INDEX IF NOT EXISTS idx_messages_room_id ON messages(room_id, id);
     CREATE INDEX IF NOT EXISTS idx_messages_channel_created ON messages(channel, created_at);
+  `);
+  const embeddingColumns = tableColumns(db, 'message_embeddings');
+  if (embeddingColumns.length > 0 && (!embeddingColumns.includes('seq') || !embeddingColumns.includes('profile') || embeddingColumns.includes('vector'))) {
+    db.exec('DROP TABLE IF EXISTS message_embeddings');
+  }
+  db.exec(`
     CREATE TABLE IF NOT EXISTS message_embeddings (
-      message_id TEXT PRIMARY KEY,
+      message_id TEXT NOT NULL,
+      seq INTEGER NOT NULL DEFAULT 0,
+      pos INTEGER NOT NULL DEFAULT 0,
       dims INTEGER NOT NULL,
-      vector TEXT NOT NULL,
-      updated_at TEXT NOT NULL
+      model TEXT NOT NULL,
+      profile TEXT NOT NULL,
+      updated_at TEXT NOT NULL,
+      PRIMARY KEY(message_id, seq)
     );
+    CREATE INDEX IF NOT EXISTS idx_message_embeddings_profile ON message_embeddings(profile);
   `);
+  ensureMessageVectorTable(db);
   try {
     db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(message_id UNINDEXED, channel UNINDEXED, sender_label, body, metadata, tokenize='porter')");
     setMeta(db, 'fts5', '1');
   } catch {
     setMeta(db, 'fts5', '0');
   }
+  setMeta(db, 'embedding_profile', EMBEDDING_PROFILE.id);
 }
 
-const VECTOR_DIMS = 256;
+let embeddingPipelinePromise = null;
+
+async function getEmbeddingPipeline() {
+  if (!embeddingPipelinePromise) {
+    embeddingPipelinePromise = (async () => {
+      const mod = await import('@huggingface/transformers');
+      const { pipeline, env } = mod;
+      if (process.env.SUPERCOLLAB_MODEL_CACHE && env) {
+        fs.mkdirSync(process.env.SUPERCOLLAB_MODEL_CACHE, { recursive: true });
+        env.cacheDir = process.env.SUPERCOLLAB_MODEL_CACHE;
+      }
+      return pipeline('feature-extraction', EMBEDDING_MODEL, { dtype: EMBEDDING_DTYPE });
+    })();
+  }
+  return embeddingPipelinePromise;
+}
 
-function tokenizeForVector(text) {
-  return String(text || '').toLowerCase().match(/[a-z0-9_./-]+/g) || [];
+function formatQueryForEmbedding(query) {
+  return `${EMBEDDING_PROFILE.query_prefix}${query}`;
 }
 
-function hashEmbedding(text) {
-  const vec = new Array(VECTOR_DIMS).fill(0);
-  for (const token of tokenizeForVector(text)) {
-    const digest = crypto.createHash('sha256').update(token).digest();
-    const idx = digest.readUInt16BE(0) % VECTOR_DIMS;
-    const sign = (digest[2] & 1) ? 1 : -1;
-    vec[idx] += sign;
+function formatDocForEmbedding(text, title = '') {
+  return title ? `${title}\n${text}` : text;
+}
+
+function chunkText(content, maxChars = EMBEDDING_CHUNK_CHARS, overlapChars = EMBEDDING_CHUNK_OVERLAP) {
+  const text = String(content || '');
+  if (text.length <= maxChars) return [{ text, pos: 0 }];
+  const chunks = [];
+  let charPos = 0;
+  while (charPos < text.length) {
+    let endPos = Math.min(charPos + maxChars, text.length);
+    if (endPos < text.length) {
+      const slice = text.slice(charPos, endPos);
+      const searchStart = Math.floor(slice.length * 0.7);
+      const searchSlice = slice.slice(searchStart);
+      let breakOffset = -1;
+      const paragraphBreak = searchSlice.lastIndexOf('\n\n');
+      if (paragraphBreak >= 0) {
+        breakOffset = searchStart + paragraphBreak + 2;
+      } else {
+        const sentenceEnd = Math.max(
+          searchSlice.lastIndexOf('. '),
+          searchSlice.lastIndexOf('.\n'),
+          searchSlice.lastIndexOf('? '),
+          searchSlice.lastIndexOf('?\n'),
+          searchSlice.lastIndexOf('! '),
+          searchSlice.lastIndexOf('!\n'),
+        );
+        if (sentenceEnd >= 0) {
+          breakOffset = searchStart + sentenceEnd + 2;
+        } else {
+          const lineBreak = searchSlice.lastIndexOf('\n');
+          if (lineBreak >= 0) {
+            breakOffset = searchStart + lineBreak + 1;
+          } else {
+            const spaceBreak = searchSlice.lastIndexOf(' ');
+            if (spaceBreak >= 0) breakOffset = searchStart + spaceBreak + 1;
+          }
+        }
+      }
+      if (breakOffset > 0) endPos = charPos + breakOffset;
+    }
+    if (endPos <= charPos) endPos = Math.min(charPos + maxChars, text.length);
+    chunks.push({ text: text.slice(charPos, endPos), pos: charPos });
+    if (endPos >= text.length) break;
+    charPos = endPos - overlapChars;
+    const lastChunkPos = chunks.at(-1).pos;
+    if (charPos <= lastChunkPos) charPos = endPos;
   }
-  const norm = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0)) || 1;
-  return vec.map((v) => Number((v / norm).toFixed(6)));
+  return chunks;
 }
 
-function cosine(a, b) {
-  let score = 0;
-  for (let i = 0; i < Math.min(a.length, b.length); i++) score += a[i] * b[i];
-  return score;
+async function embedText(text, { isQuery = false, title = '' } = {}) {
+  const extractor = await getEmbeddingPipeline();
+  const formatted = isQuery ? formatQueryForEmbedding(text) : formatDocForEmbedding(text, title);
+  const output = await extractor(formatted.slice(0, 4000), {
+    pooling: EMBEDDING_PROFILE.pooling,
+    normalize: EMBEDDING_PROFILE.normalize,
+  });
+  const vector = Array.from(output.data).map(Number);
+  if (vector.length !== EMBEDDING_DIMS) throw new Error(`unexpected embedding dims ${vector.length}`);
+  return vector;
+}
+
+async function storeEmbeddings(db, local, metadata) {
+  const messageId = local.message_id;
+  if (!messageId) return { embedded: false, chunks: 0 };
+  const oldRows = dbAll(db, 'SELECT seq FROM message_embeddings WHERE message_id=? AND profile<>?', [messageId, EMBEDDING_PROFILE.id]);
+  for (const row of oldRows) dbRun(db, 'DELETE FROM message_vectors WHERE message_seq=?', [`${messageId}:${Number(row.seq || 0)}`]);
+  dbRun(db, 'DELETE FROM message_embeddings WHERE message_id=? AND profile<>?', [messageId, EMBEDDING_PROFILE.id]);
+  const existing = dbGet(
+    db,
+    `SELECT COUNT(*) AS count
+     FROM message_embeddings e
+     JOIN message_vectors v ON v.message_seq = e.message_id || ':' || e.seq
+     WHERE e.message_id=? AND e.profile=?`,
+    [messageId, EMBEDDING_PROFILE.id],
+  );
+  if (Number(existing?.count || 0) > 0) return { embedded: false, chunks: Number(existing.count) };
+
+  const body = `${local.sender_label || ''}\n${local.body || ''}\n${metadata || ''}`;
+  const title = local.sender_label || local.channel || 'SuperCollab message';
+  const chunks = chunkText(body);
+  const updatedAt = nowIso();
+  for (let seq = 0; seq < chunks.length; seq++) {
+    const chunk = chunks[seq];
+    const vector = await embedText(chunk.text, { title });
+    const messageSeq = `${messageId}:${seq}`;
+    dbRun(db, 'INSERT OR REPLACE INTO message_vectors(message_seq, embedding) VALUES(?, ?)', [messageSeq, new Float32Array(vector)]);
+    dbRun(
+      db,
+      `INSERT INTO message_embeddings(message_id,seq,pos,dims,model,profile,updated_at)
+       VALUES(?,?,?,?,?,?,?)
+       ON CONFLICT(message_id, seq) DO UPDATE SET
+         pos=excluded.pos,
+         dims=excluded.dims,
+         model=excluded.model,
+         profile=excluded.profile,
+         updated_at=excluded.updated_at`,
+      [messageId, seq, chunk.pos, EMBEDDING_DIMS, EMBEDDING_MODEL, EMBEDDING_PROFILE.id, updatedAt],
+    );
+  }
+  setMeta(db, 'embedding_last_ok_at', updatedAt);
+  return { embedded: true, chunks: chunks.length };
 }
 
-function storeEmbedding(db, messageId, text) {
-  const vector = hashEmbedding(text);
-  dbRun(
+async function tryStoreEmbeddings(db, local, metadata) {
+  try {
+    return await storeEmbeddings(db, local, metadata);
+  } catch (err) {
+    setMeta(db, 'embedding_last_error', err.message || String(err));
+    return { embedded: false, chunks: 0, error: err.message || String(err) };
+  }
+}
+
+async function embedMissingMessages(db, limit = 500) {
+  const rows = dbAll(
     db,
-    'INSERT INTO message_embeddings(message_id,dims,vector,updated_at) VALUES(?,?,?,?) ON CONFLICT(message_id) DO UPDATE SET dims=excluded.dims, vector=excluded.vector, updated_at=excluded.updated_at',
-    [messageId, VECTOR_DIMS, JSON.stringify(vector), nowIso()],
+    `SELECT m.*
+     FROM messages m
+     LEFT JOIN message_embeddings e
+       ON e.message_id=m.message_id AND e.profile=?
+      LEFT JOIN message_vectors v
+        ON v.message_seq = e.message_id || ':' || e.seq
+     WHERE e.message_id IS NULL OR v.message_seq IS NULL
+     ORDER BY m.id ASC
+     LIMIT ?`,
+    [EMBEDDING_PROFILE.id, Math.max(1, Math.min(Number(limit || 500), 2000))],
   );
+  let embedded = 0;
+  for (const row of rows) {
+    const result = await tryStoreEmbeddings(db, row, row.metadata || '');
+    if (result.embedded) embedded += result.chunks;
+  }
+  return { messages_checked: rows.length, chunks_embedded: embedded };
 }
 
 async function openChatDb(config, file, roomId) {
-  const SQL = await loadSqlJs();
+  const { Database, sqliteVec } = await loadNativeSqlite();
   const root = chatRoot(config, file, roomId);
   const dbPath = chatDbPath(config, file, roomId);
   fs.mkdirSync(root, { recursive: true, mode: 0o700 });
-  let db;
-  if (fs.existsSync(dbPath)) {
-    db = new SQL.Database(fs.readFileSync(dbPath));
-  } else {
-    db = new SQL.Database();
-  }
+  const db = new Database(dbPath);
+  db.pragma('journal_mode = WAL');
+  db.pragma('foreign_keys = ON');
+  sqliteVec.load(db);
+  verifySqliteVecLoaded(db);
   initChatSchema(db);
   setMeta(db, 'room_id', roomId);
   return { db, root, dbPath, roomId };
 }
 
 function saveChatDb(cap) {
-  const tmp = `${cap.dbPath}.${process.pid}.tmp`;
-  fs.writeFileSync(tmp, Buffer.from(cap.db.export()), { mode: 0o600 });
-  fs.renameSync(tmp, cap.dbPath);
+  cap.db.pragma('wal_checkpoint(PASSIVE)');
   try { fs.chmodSync(cap.dbPath, 0o600); } catch {}
 }
 
@@ -512,7 +683,7 @@ function localPlainMessage(config, roomId, msg) {
   return { ...msg, metadata: JSON.stringify(metadata) };
 }
 
-function insertLocalMessage(db, msg, config = null, roomId = msg.room_id || '') {
+async function insertLocalMessage(db, msg, config = null, roomId = msg.room_id || '') {
   const local = config ? localPlainMessage(config, roomId, msg) : msg;
   const metadata = typeof local.metadata === 'string' ? local.metadata : JSON.stringify(local.metadata || {});
   dbRun(
@@ -532,7 +703,7 @@ function insertLocalMessage(db, msg, config = null, roomId = msg.room_id || '')
       ]);
     } catch {}
   }
-  storeEmbedding(db, local.message_id, `${local.sender_label || ''}\n${local.body || ''}\n${metadata}`);
+  await tryStoreEmbeddings(db, local, metadata);
 }
 
 async function syncRoom(config, file, roomId, limit = 500) {
@@ -540,11 +711,12 @@ async function syncRoom(config, file, roomId, limit = 500) {
   try {
     const after = Number(getMeta(cap.db, 'last_message_id', '0')) || 0;
     const data = await apiAsAgent(config, 'GET', `/v1/rooms/${roomId}/messages?after=${encodeURIComponent(after)}&limit=${encodeURIComponent(limit)}`);
-    for (const msg of data.messages || []) insertLocalMessage(cap.db, { ...msg, room_id: roomId }, config, roomId);
+    for (const msg of data.messages || []) await insertLocalMessage(cap.db, { ...msg, room_id: roomId }, config, roomId);
+    const embedding = await embedMissingMessages(cap.db, 500);
     setMeta(cap.db, 'last_message_id', String(data.next_after || after));
     setMeta(cap.db, 'last_sync_at', nowIso());
     saveChatDb(cap);
-    return { room_id: roomId, pulled: (data.messages || []).length, last_message_id: Number(data.next_after || after), db: cap.dbPath };
+    return { room_id: roomId, pulled: (data.messages || []).length, last_message_id: Number(data.next_after || after), db: cap.dbPath, embedding };
   } finally {
     cap.db.close();
   }
@@ -605,7 +777,7 @@ async function doChatSend(config, file, opts) {
   });
   const cap = await openChatDb(config, file, roomId);
   try {
-    insertLocalMessage(cap.db, { ...data.message, room_id: roomId }, config, roomId);
+    await insertLocalMessage(cap.db, { ...data.message, room_id: roomId }, config, roomId);
     setMeta(cap.db, 'last_message_id', String(Math.max(Number(getMeta(cap.db, 'last_message_id', '0')) || 0, Number(data.message.id))));
     saveChatDb(cap);
   } finally {
@@ -634,16 +806,19 @@ function ftsQuery(value) {
 async function doChatSearch(config, file, opts) {
   const roomId = requireValue(opts, 'room');
   const query = requireValue(opts, 'query');
+  const mode = String(opts.mode || 'hybrid').toLowerCase();
+  if (!['hybrid', 'keyword', 'vector'].includes(mode)) throw new Error('search --mode must be hybrid, keyword, or vector');
   await syncRoom(config, file, roomId, 500);
   const cap = await openChatDb(config, file, roomId);
   try {
+    const embedding = await embedMissingMessages(cap.db, 500);
     const maxResults = Math.max(1, Math.min(Number(opts.limit || 20), 100));
-    let rows = [];
-    if (getMeta(cap.db, 'fts5', '0') === '1') {
+    let keywordRows = [];
+    if (mode !== 'vector' && getMeta(cap.db, 'fts5', '0') === '1') {
       const q = ftsQuery(query);
       if (q) {
         try {
-          rows = dbAll(
+          keywordRows = dbAll(
             cap.db,
             `SELECT m.*, bm25(messages_fts) AS score
              FROM messages_fts JOIN messages m ON m.id=messages_fts.rowid
@@ -652,33 +827,96 @@ async function doChatSearch(config, file, opts) {
             [q, maxResults],
           );
         } catch {
-          rows = [];
+          keywordRows = [];
         }
       }
     }
-    if (!rows.length) {
-      rows = dbAll(cap.db, 'SELECT *, 0 AS score FROM messages WHERE body LIKE ? OR metadata LIKE ? ORDER BY id DESC LIMIT ?', [
+    if (mode !== 'vector' && !keywordRows.length) {
+      keywordRows = dbAll(cap.db, 'SELECT *, 0 AS score FROM messages WHERE body LIKE ? OR metadata LIKE ? ORDER BY id DESC LIMIT ?', [
         `%${query}%`, `%${query}%`, maxResults,
-      ]);
+      ]).map((row) => ({ ...row, keyword_fallback: true }));
+    }
+    let vectorRows = [];
+    let vectorError = null;
+    if (mode !== 'keyword') try {
+      const qvec = await embedText(query, { isQuery: true });
+      const k = Math.max(maxResults * 4, 50);
+      const vecMatches = dbAll(
+        cap.db,
+        'SELECT message_seq, distance FROM message_vectors WHERE embedding MATCH ? AND k = ?',
+        [new Float32Array(qvec), k],
+      );
+      const bestByMessage = new Map();
+      if (vecMatches.length) {
+        const messageSeqs = vecMatches.map((row) => String(row.message_seq));
+        const distanceBySeq = new Map(vecMatches.map((row) => [String(row.message_seq), Number(row.distance)]));
+        const placeholders = messageSeqs.map(() => '?').join(',');
+        for (const row of dbAll(
+          cap.db,
+          `SELECT m.*, e.seq, e.pos, e.message_id || ':' || e.seq AS message_seq
+           FROM message_embeddings e
+           JOIN messages m ON m.message_id=e.message_id
+           WHERE e.profile=? AND e.message_id || ':' || e.seq IN (${placeholders})`,
+          [EMBEDDING_PROFILE.id, ...messageSeqs],
+        )) {
+          const distance = distanceBySeq.get(String(row.message_seq)) ?? 1;
+          const score = 1 - distance;
+          if (score <= 0) continue;
+          const { message_seq, ...clean } = row;
+          const prior = bestByMessage.get(row.message_id);
+          if (!prior || score > prior.vector_score) {
+            bestByMessage.set(row.message_id, { ...clean, vector_score: score, chunk_seq: Number(row.seq || 0), chunk_pos: Number(row.pos || 0) });
+          }
+        }
+      }
+      vectorRows = Array.from(bestByMessage.values())
+        .sort((a, b) => b.vector_score - a.vector_score)
+        .slice(0, mode === 'vector' ? maxResults : Math.max(maxResults, 50));
+    } catch (err) {
+      vectorError = err.message || String(err);
+      setMeta(cap.db, 'embedding_last_error', vectorError);
+    }
+    const keywordRank = new Map(keywordRows.map((row, idx) => [row.message_id, idx + 1]));
+    const vectorRank = new Map(vectorRows.map((row, idx) => [row.message_id, idx + 1]));
+    const byMessage = new Map();
+    for (const row of [...keywordRows, ...vectorRows]) {
+      const existing = byMessage.get(row.message_id) || {};
+      byMessage.set(row.message_id, { ...existing, ...row });
     }
-    const seen = new Set(rows.map((row) => row.message_id));
-    const qvec = hashEmbedding(query);
-    const vectorRows = dbAll(
-      cap.db,
-      `SELECT m.*, e.vector
-       FROM message_embeddings e JOIN messages m ON m.message_id=e.message_id
-       ORDER BY m.id DESC LIMIT 1000`,
-    )
-      .map((row) => {
-        let score = 0;
-        try { score = cosine(qvec, JSON.parse(row.vector)); } catch {}
-        const { vector, ...clean } = row;
-        return { ...clean, vector_score: score };
-      })
-      .filter((row) => row.vector_score > 0 && !seen.has(row.message_id))
-      .sort((a, b) => b.vector_score - a.vector_score)
-      .slice(0, Math.max(0, maxResults - rows.length));
-    return { room_id: roomId, query, search: { local_only: true, fts: true, vector: 'hash-256' }, results: [...rows, ...vectorRows] };
+    const rrfK = 60;
+    const hybridRows = Array.from(byMessage.values()).map((row) => {
+      const kr = keywordRank.get(row.message_id);
+      const vr = vectorRank.get(row.message_id);
+      const keywordScore = kr ? 1 / (rrfK + kr) : 0;
+      const vectorScore = vr ? 1 / (rrfK + vr) : 0;
+      return {
+        ...row,
+        search_sources: [kr ? (row.keyword_fallback ? 'like' : 'fts5_bm25') : null, vr ? 'bge_vector_cosine' : null].filter(Boolean),
+        keyword_rank: kr || null,
+        vector_rank: vr || null,
+        hybrid_score: keywordScore + vectorScore,
+      };
+    }).sort((a, b) => b.hybrid_score - a.hybrid_score).slice(0, maxResults);
+    const results = mode === 'keyword'
+      ? keywordRows.slice(0, maxResults).map((row, idx) => ({ ...row, search_sources: [row.keyword_fallback ? 'like' : 'fts5_bm25'], keyword_rank: idx + 1 }))
+      : mode === 'vector'
+        ? vectorRows.slice(0, maxResults).map((row, idx) => ({ ...row, search_sources: ['bge_vector_cosine'], vector_rank: idx + 1 }))
+        : hybridRows;
+    return {
+      room_id: roomId,
+      query,
+      search: {
+        local_only: true,
+        mode,
+        methods: ['fts5_bm25', 'bge_vector_cosine', 'rrf_hybrid'],
+        fts: getMeta(cap.db, 'fts5', '0') === '1',
+        vector: EMBEDDING_PROFILE.id,
+        embedding_profile: EMBEDDING_PROFILE,
+        embedding,
+        vector_error: vectorError,
+      },
+      results,
+    };
   } finally {
     cap.db.close();
   }
@@ -737,6 +975,7 @@ async function activeStatus(config, file, opts = {}) {
     room_id: active?.roomId || null,
     activation_root: active?.cwd || null,
     config: file,
+    embedding_profile: EMBEDDING_PROFILE,
     instructions: agentInstructions(active),
   };
 }
@@ -764,7 +1003,7 @@ function mcpTools() {
     toolSchema('room_join', 'Accept a room invite token.', { invite_token: s, fingerprint: s }, ['invite_token']),
     toolSchema('chat_send', 'Send a message to the active agent chat room.', { text: s, channel: s, kind: s }, ['text']),
     toolSchema('chat_read', 'Sync and read recent messages from the active room.', { limit: { type: 'integer' } }),
-    toolSchema('chat_search', 'Sync and search the active room transcript.', { query: s, limit: { type: 'integer' } }, ['query']),
+    toolSchema('chat_search', 'Sync and search the active room transcript with local keyword, BGE vector, or hybrid retrieval.', { query: s, mode: s, limit: { type: 'integer' } }, ['query']),
     toolSchema('chat_sync', 'Sync the active room transcript into local SQLite.'),
   ];
 }
@@ -778,7 +1017,7 @@ async function callTool(config, name, args) {
   if (name === 'room_join') return doRoomJoin(config, file, { invite: args.invite_token });
   if (name === 'chat_send') return doChatSend(config, file, { room: requireActiveRoom(config, args), text: args.text, channel: args.channel || 'agents', kind: args.kind || 'chat.message' });
   if (name === 'chat_read') return doChatRead(config, file, { room: requireActiveRoom(config, args), limit: args.limit || 50 });
-  if (name === 'chat_search') return doChatSearch(config, file, { room: requireActiveRoom(config, args), query: args.query, limit: args.limit || 20 });
+  if (name === 'chat_search') return doChatSearch(config, file, { room: requireActiveRoom(config, args), query: args.query, mode: args.mode || 'hybrid', limit: args.limit || 20 });
   if (name === 'chat_sync') return syncRoom(config, file, requireActiveRoom(config, args));
   throw new Error(`unknown tool: ${name}`);
 }
@@ -858,6 +1097,24 @@ function printCodexConfig(opts) {
   console.log(`[mcp_servers.supercollab]\ncommand = "supercollab"\nargs = ["mcp", "stdio", "--config", "${file.replaceAll('\\', '\\\\').replaceAll('"', '\\"')}"]`);
 }
 
+async function embeddingStatus() {
+  return {
+    ok: true,
+    profile: EMBEDDING_PROFILE,
+    model_download: 'lazy on first embedding, or now via `supercollab embeddings warmup`',
+    cache_dir: process.env.SUPERCOLLAB_MODEL_CACHE || 'default @huggingface/transformers cache',
+  };
+}
+
+async function embeddingWarmup() {
+  const vector = await embedText('supercollab embedding warmup', { isQuery: true });
+  return {
+    ok: true,
+    dims: vector.length,
+    profile: EMBEDDING_PROFILE,
+  };
+}
+
 async function main() {
   const { positionals, opts } = parse(process.argv.slice(2));
   if (opts.help || positionals.length === 0) { printHelp(); return; }
@@ -896,6 +1153,10 @@ async function main() {
     if (sub === 'list') return console.log(JSON.stringify(await api(config, 'GET', '/v1/agent-sessions', undefined, config.userToken), null, 2));
     if (sub === 'revoke') return console.log(JSON.stringify(await api(config, 'DELETE', `/v1/agent-sessions/${requireValue(opts, 'session')}`, undefined, config.userToken), null, 2));
   }
+  if (cmd === 'embeddings') {
+    if (sub === 'status') return console.log(JSON.stringify(await embeddingStatus(), null, 2));
+    if (sub === 'warmup') return console.log(JSON.stringify(await embeddingWarmup(), null, 2));
+  }
   if (cmd === 'mcp' && sub === 'stdio') return runMcp(opts);
   if (cmd === 'mcp' && sub === 'print-config') return printCodexConfig(opts);
   throw new Error(`unknown command: ${positionals.join(' ')}`);

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@supercollab/cli",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "description": "SuperCollab CLI and MCP bridge for encrypted local-search agent group chat.",
   "type": "module",
   "bin": {
-    "supercollab": "./bin/supercollab.js"
+    "supercollab": "bin/supercollab.js"
   },
   "files": [
     "bin/supercollab.js",
@@ -14,7 +14,9 @@
     "node": ">=20"
   },
   "dependencies": {
-    "sql.js": "^1.14.1"
+    "@huggingface/transformers": "3.8.1",
+    "better-sqlite3": "12.11.1",
+    "sqlite-vec": "0.1.9"
   },
   "keywords": [
     "mcp",