@wirecat/cli-core 0.433.0 → 0.435.0

package/dist/store/sqlite/image-chunks.js

@@ -1,4 +1,5 @@
 import { sql } from "./drizzle/core.js";
+import { dropUnusedVectors, searchIndex } from "./vector-index.js";
 import { dot } from "./vectors.js";
 const IS_IMAGE = sql `(a.kind = 'photo' OR a.mime LIKE 'image/%')`;
 const neighbours = ({ orm }, chatKey, messagePk, sentAt, { messages, withinMs }, side) => orm.all(sql `SELECT sender_name AS sender, text, sent_at AS sentAt FROM messages
@@ -26,7 +27,7 @@ export const imagesOfChat = (context, chatKey, window) => context.orm
  * Makes these the chunks of their images, and drops the chunks of images that are gone or whose message was
  * deleted, with the vectors nothing else uses any more.
  */
-export const saveImageChunks = ({ orm, now }, accountPk, chatKey, chunks, keep) => {
+export const saveImageChunks = ({ orm, now, database }, accountPk, chatKey, chunks, keep) => {
     const at = now();
     const scope = orm.get(sql `SELECT scope FROM accounts WHERE id = ${accountPk}`)?.scope ?? null;
     const old = new Set();
@@ -50,9 +51,7 @@ export const saveImageChunks = ({ orm, now }, accountPk, chatKey, chunks, keep)
           account_id, occurred_at, created_at, updated_at)
         VALUES ('image', ${attachmentPk}, 0, 0, ${length}, ${hash}, ${scope}, ${accountPk}, ${occurredAt}, ${at}, ${at})`);
     }
-    for (const hash of old)
-        orm.run(sql `DELETE FROM embeddings WHERE content_hash = ${hash}
-      AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ${hash})`);
+    dropUnusedVectors(database, old);
 };
 /** The hashes among these that already have a vector of `model`. */
 export const embeddedHashes = ({ orm }, model, hashes) => new Set(hashes.length === 0
@@ -62,32 +61,64 @@ export const embeddedHashes = ({ orm }, model, hashes) => new Set(hashes.length
               WHERE model = ${model} AND content_hash IN (SELECT value FROM json_each(${JSON.stringify(hashes)}))`)
         .map(({ hash }) => hash));
 const SCAN_PAGE = 5_000;
+/** The scan's rows: one page after `after`, or the `ids` the index picked. */
+const imageRows = ({ orm }, accountPk, { chatKey, model, since, position }, rows) => orm.all(sql `SELECT k.id, k.content_hash AS hash, ch.external_id AS chatId, m.external_id AS messageId, v.vector
+      FROM chunks k CROSS JOIN attachments a ON a.id = k.chunkable_id AND a.attachable_type = 'message'
+        JOIN messages m ON m.id = a.attachable_id AND m.deleted_at IS NULL
+        JOIN chats ch ON ch.id = m.chat_id
+        JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
+      WHERE k.chunkable_type = 'image' AND k.position = ${position} AND ch.account_id = ${accountPk}
+        ${chatKey === undefined ? sql `AND ch.searchable = 1` : sql `AND m.chat_id = ${chatKey}`}
+        ${since === undefined ? sql `` : sql `AND k.occurred_at >= ${since}`}
+        ${"ids" in rows
+    ? sql `AND k.id IN (SELECT value FROM json_each(${JSON.stringify(rows.ids)})) ORDER BY k.id`
+    : sql `AND k.id > ${rows.after} ORDER BY k.id LIMIT ${SCAN_PAGE}`}`);
 /**
  * The images of the account nearest to `query` by the vector of `model`, best first. Position 0 of an image is
  * its context text, position 1 its picture (`PICTURE_POSITION`): a model's vectors only ever sit at one.
  */
-export const nearestImages = ({ orm }, accountPk, { chatKey, model, since, limit, query, position = 0, }) => {
+export const nearestImages = (context, accountPk, { chatKey, model, since, limit, query, position = 0, }) => {
+    const filter = {
+        model,
+        position,
+        ...(chatKey === undefined ? {} : { chatKey }),
+        ...(since === undefined ? {} : { since }),
+    };
+    const score = (row) => ({
+        id: row.id,
+        chatId: row.chatId,
+        messageId: row.messageId,
+        score: dot(query, row.vector),
+    });
+    // Stable, so equal scores keep the scan's order, by chunk id.
+    const best = (found) => found.sort((a, b) => b.score - a.score).slice(0, limit);
+    const indexed = searchIndex(context.database, model, {
+        kind: "image",
+        account: accountPk,
+        ...(chatKey === undefined ? {} : { chat: chatKey }),
+        ...(since === undefined ? {} : { since }),
+    }, query, limit, (candidates) => {
+        const hashes = new Map(candidates.map(({ id, hash }) => [id, hash]));
+        const results = best(imageRows(context, accountPk, filter, { ids: [...hashes.keys()] })
+            .filter((row) => hashes.get(row.id) === row.hash)
+            .map(score));
+        return { results, best: results.map(({ id }) => id) };
+    });
+    const plain = (found) => found.map(({ chatId, messageId, score }) => ({ chatId, messageId, score }));
+    if (indexed)
+        return plain(indexed);
     const found = [];
     let after = 0;
     for (;;) {
-        const rows = orm.all(sql `SELECT k.id, ch.external_id AS chatId, m.external_id AS messageId, v.vector
-        FROM chunks k CROSS JOIN attachments a ON a.id = k.chunkable_id AND a.attachable_type = 'message'
-          JOIN messages m ON m.id = a.attachable_id AND m.deleted_at IS NULL
-          JOIN chats ch ON ch.id = m.chat_id
-          JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
-        WHERE k.chunkable_type = 'image' AND k.position = ${position} AND ch.account_id = ${accountPk}
-          AND k.id > ${after}
-          ${chatKey === undefined ? sql `AND ch.searchable = 1` : sql `AND m.chat_id = ${chatKey}`}
-          ${since === undefined ? sql `` : sql `AND k.occurred_at >= ${since}`}
-        ORDER BY k.id LIMIT ${SCAN_PAGE}`);
+        const rows = imageRows(context, accountPk, filter, { after });
         for (const row of rows)
-            found.push({ chatId: row.chatId, messageId: row.messageId, score: dot(query, row.vector) });
+            found.push(score(row));
         const last = rows.at(-1);
         if (!last || rows.length < SCAN_PAGE)
             break;
         after = last.id;
     }
-    return found.sort((a, b) => b.score - a.score).slice(0, limit);
+    return plain(best(found));
 };
 /** An image's own vector is its chunk at this position; position 0 is the text around it. */
 export const PICTURE_POSITION = 1;
@@ -102,24 +133,23 @@ export const picturesToEmbed = ({ orm }, accountPk, { chatKey, after, limit }) =
           AND k.position = ${PICTURE_POSITION})
       ORDER BY a.id LIMIT ${limit}`);
 /** Makes each hash the picture chunk of its image; the vectors are saved apart, by `saveVectors`. */
-export const savePictureChunks = ({ orm, now }, accountPk, pictures) => {
+export const savePictureChunks = ({ orm, now, database }, accountPk, pictures) => {
     const at = now();
     const scope = orm.get(sql `SELECT scope FROM accounts WHERE id = ${accountPk}`)?.scope ?? null;
     // A deleted message keeps no trace of its picture, as it keeps none of its text.
     const gone = sql `chunkable_type = 'image' AND position = ${PICTURE_POSITION} AND account_id = ${accountPk}
     AND NOT EXISTS (SELECT 1 FROM attachments a JOIN messages m ON a.attachable_type = 'message'
       AND m.id = a.attachable_id WHERE a.id = chunkable_id AND m.deleted_at IS NULL)`;
-    const dropped = orm.all(sql `SELECT content_hash AS hash FROM chunks WHERE ${gone}`);
-    orm.run(sql `DELETE FROM chunks WHERE ${gone}`);
-    for (const { hash } of dropped)
-        orm.run(sql `DELETE FROM embeddings WHERE content_hash = ${hash}
-      AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ${hash})`);
+    const released = orm.all(sql `DELETE FROM chunks WHERE ${gone} RETURNING content_hash AS hash`);
     for (const { attachmentPk, hash, bytes, occurredAt } of pictures) {
-        orm.run(sql `DELETE FROM chunks WHERE chunkable_type = 'image' AND chunkable_id = ${attachmentPk}
-        AND position = ${PICTURE_POSITION}`);
+        const replaced = orm.get(sql `DELETE FROM chunks WHERE chunkable_type = 'image' AND chunkable_id = ${attachmentPk}
+        AND position = ${PICTURE_POSITION} RETURNING content_hash AS hash`);
+        if (replaced)
+            released.push(replaced);
         orm.run(sql `INSERT INTO chunks (chunkable_type, chunkable_id, position, start_offset, end_offset, content_hash, scope,
           account_id, occurred_at, created_at, updated_at)
         VALUES ('image', ${attachmentPk}, ${PICTURE_POSITION}, 0, ${bytes}, ${hash}, ${scope}, ${accountPk},
           ${occurredAt}, ${at}, ${at})`);
     }
+    dropUnusedVectors(database, released.map(({ hash }) => hash));
 };

package/dist/store/sqlite/manifest.js

@@ -43,6 +43,8 @@ export const MANIFEST = [
     { name: "20261025071850_version-32-files-data", version: 32, minCompatible: 32 },
     { name: "20261025072643_version-33-calendars", version: 33, minCompatible: 32 },
     { name: "20261025072644_version-33-calendars-triggers", version: 33, minCompatible: 32 },
+    { name: "20261025182500_version-34-vector-index", version: 34, minCompatible: 34 },
+    { name: "20261025182501_version-34-vector-index-triggers", version: 34, minCompatible: 34 },
 ];
 export const generatedMigrations = (manifest = MANIFEST, generated = GENERATED) => {
     const migrations = [];

package/dist/store/sqlite/meeting-vectors.js

@@ -3,6 +3,7 @@ import { chunkHash, cutChunks } from "../../conversations/chunks.js";
 import { positiveReadInteger } from "./meeting-reads.js";
 import { meetingSelectionSql } from "./meeting-selection.js";
 import { read, write } from "./transaction.js";
+import { dropUnusedVectors } from "./vector-index.js";
 import { dot, saveVectors } from "./vectors.js";
 /** Vectors read per query while ranking, so a store of any size stays within a few megabytes at a time. */
 const RANK_BATCH = 512;
@@ -129,11 +130,13 @@ export const meetingVectorsOver = (context) => ({
                 text: load(Number(row.id)),
             }));
             let count = 0;
+            const released = [];
             for (const { row, text } of prepared) {
                 input.signal?.throwIfAborted();
-                context.database
-                    .prepare("DELETE FROM chunks WHERE chunkable_type='meeting_transcript' AND chunkable_id=?")
-                    .run(Number(row.id));
+                for (const { hash } of context.database
+                    .prepare("DELETE FROM chunks WHERE chunkable_type='meeting_transcript' AND chunkable_id=? RETURNING content_hash AS hash")
+                    .all(Number(row.id)))
+                    released.push(String(hash));
                 const chunks = cutChunks([{ id: String(row.id), sender: null, text: text.text }], undefined, () => input.signal?.throwIfAborted());
                 for (let index = 0; index < chunks.length; index++) {
                     const chunk = chunks[index];
@@ -145,12 +148,11 @@ export const meetingVectorsOver = (context) => ({
                     count++;
                 }
             }
-            context.database
-                .prepare("DELETE FROM embeddings WHERE EXISTS (SELECT 1 FROM chunks k WHERE k.content_hash=embeddings.content_hash AND k.chunkable_type='meeting_transcript' AND k.account_id=? AND NOT EXISTS (SELECT 1 FROM meeting_transcripts t JOIN meetings m ON m.id=t.meeting_id WHERE t.id=k.chunkable_id AND t.deleted_at IS NULL AND t.superseded_at IS NULL AND m.deleted_at IS NULL AND m.account_id=?)) AND NOT EXISTS (SELECT 1 FROM chunks k WHERE k.content_hash=embeddings.content_hash AND (k.chunkable_type <> 'meeting_transcript' OR k.account_id IS NOT ? OR EXISTS (SELECT 1 FROM meeting_transcripts t JOIN meetings m ON m.id=t.meeting_id WHERE t.id=k.chunkable_id AND t.deleted_at IS NULL AND t.superseded_at IS NULL AND m.deleted_at IS NULL AND m.account_id=?)))")
-                .run(input.accountId, input.accountId, input.accountId, input.accountId);
-            context.database
-                .prepare("DELETE FROM chunks WHERE chunkable_type='meeting_transcript' AND account_id=? AND NOT EXISTS (SELECT 1 FROM meeting_transcripts t JOIN meetings m ON m.id=t.meeting_id WHERE t.id=chunks.chunkable_id AND t.deleted_at IS NULL AND t.superseded_at IS NULL AND m.deleted_at IS NULL AND m.account_id=?)")
-                .run(input.accountId, input.accountId);
+            for (const { hash } of context.database
+                .prepare("DELETE FROM chunks WHERE chunkable_type='meeting_transcript' AND account_id=? AND NOT EXISTS (SELECT 1 FROM meeting_transcripts t JOIN meetings m ON m.id=t.meeting_id WHERE t.id=chunks.chunkable_id AND t.deleted_at IS NULL AND t.superseded_at IS NULL AND m.deleted_at IS NULL AND m.account_id=?) RETURNING content_hash AS hash")
+                .all(input.accountId, input.accountId))
+                released.push(String(hash));
+            dropUnusedVectors(context.database, released);
             input.signal?.throwIfAborted();
             const hasMore = found.length > limit;
             return {

package/dist/store/sqlite/message-chunks.d.ts

@@ -33,12 +33,15 @@ export interface NearestMessage {
     messageId: string;
     score: number;
 }
-/** The account's live messages nearest to `query` by their vector of `model`, best first: a scan of every vector. */
-export declare const nearestMessages: ({ orm }: StoreContext, accountPk: number, { chatKey, model, since, before, limit, query, }: {
+type MessageFilter = {
     chatKey?: number;
     model: string;
     since?: number;
     before?: number;
+};
+/** The account's live messages nearest to `query` by their vector of `model`, best first. */
+export declare const nearestMessages: (context: StoreContext, accountPk: number, { limit, query, ...filter }: MessageFilter & {
     limit: number;
     query: Float32Array;
 }) => NearestMessage[];
+export {};

package/dist/store/sqlite/message-chunks.js

@@ -1,4 +1,5 @@
 import { sql } from "./drizzle/core.js";
+import { dropUnusedVectors, searchIndex } from "./vector-index.js";
 import { dot } from "./vectors.js";
 /** Live messages with text of the account, by pk after `after`; the caller compares hashes. */
 export const messagesToEmbed = ({ orm }, accountPk, { chatKey, model, after, limit }) => orm
@@ -10,11 +11,6 @@ export const messagesToEmbed = ({ orm }, accountPk, { chatKey, model, after, lim
           ${chatKey === undefined ? sql `` : sql `AND m.chat_id = ${chatKey}`}
         ORDER BY m.id LIMIT ${limit}`)
     .map((row) => ({ ...row, embedded: row.embedded === 1 }));
-const dropUnshared = ({ orm }, hashes) => {
-    for (const hash of hashes)
-        orm.run(sql `DELETE FROM embeddings WHERE content_hash = ${hash}
-      AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ${hash})`);
-};
 /** Makes each hash its message's chunk; the vectors are saved apart, by `saveVectors`. */
 export const saveMessageChunks = (context, accountPk, chunks) => {
     const { orm, now } = context;
@@ -33,7 +29,7 @@ export const saveMessageChunks = (context, accountPk, chunks) => {
           account_id, occurred_at, created_at, updated_at)
         VALUES ('message', ${messagePk}, 0, 0, ${length}, ${hash}, ${scope}, ${accountPk}, ${occurredAt}, ${at}, ${at})`);
     }
-    dropUnshared(context, old);
+    dropUnusedVectors(context.database, old);
 };
 /**
  * Drops the chunks of the account's messages that are gone, deleted or emptied, with the vectors nothing else
@@ -44,33 +40,61 @@ export const dropGoneMessageChunks = (context, accountPk) => {
     WHERE m.id = chunkable_id AND m.deleted_at IS NULL AND trim(m.text) <> '')`;
     const dropped = context.orm.all(sql `SELECT content_hash AS hash FROM chunks WHERE ${gone}`);
     context.orm.run(sql `DELETE FROM chunks WHERE ${gone}`);
-    dropUnshared(context, dropped.map(({ hash }) => hash));
+    dropUnusedVectors(context.database, dropped.map(({ hash }) => hash));
     return dropped.length;
 };
 /** Whether any message of the account has a vector of `model`. */
 export const hasMessageVectors = ({ orm }, accountPk, model) => orm.get(sql `SELECT 1 FROM chunks k JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
       WHERE k.chunkable_type = 'message' AND k.account_id = ${accountPk} LIMIT 1`) !== undefined;
 const SCAN_PAGE = 5_000;
-/** The account's live messages nearest to `query` by their vector of `model`, best first: a scan of every vector. */
-export const nearestMessages = ({ orm }, accountPk, { chatKey, model, since, before, limit, query, }) => {
+/** The scan's rows: one page after `after`, or the `ids` the index picked. */
+const messageRows = ({ orm }, accountPk, { chatKey, model, since, before }, rows) => orm.all(sql `SELECT k.id, k.content_hash AS hash, ch.external_id AS chatId, m.external_id AS messageId, v.vector
+      FROM chunks k CROSS JOIN messages m ON m.id = k.chunkable_id AND m.deleted_at IS NULL
+        JOIN chats ch ON ch.id = m.chat_id
+        JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
+      WHERE k.chunkable_type = 'message' AND k.position = 0 AND k.account_id = ${accountPk}
+        ${chatKey === undefined ? sql `AND ch.searchable = 1` : sql `AND m.chat_id = ${chatKey}`}
+        ${since === undefined ? sql `` : sql `AND m.sent_at >= ${since}`}
+        ${before === undefined ? sql `` : sql `AND m.sent_at < ${before}`}
+        ${"ids" in rows
+    ? sql `AND k.id IN (SELECT value FROM json_each(${JSON.stringify(rows.ids)})) ORDER BY k.id`
+    : sql `AND k.id > ${rows.after} ORDER BY k.id LIMIT ${SCAN_PAGE}`}`);
+/** The account's live messages nearest to `query` by their vector of `model`, best first. */
+export const nearestMessages = (context, accountPk, { limit, query, ...filter }) => {
+    const score = (row) => ({
+        id: row.id,
+        chatId: row.chatId,
+        messageId: row.messageId,
+        score: dot(query, row.vector),
+    });
+    // Stable, so equal scores keep the scan's order, by chunk id.
+    const best = (found) => found.sort((a, b) => b.score - a.score).slice(0, limit);
+    const indexed = searchIndex(context.database, filter.model, {
+        kind: "message",
+        account: accountPk,
+        ...(filter.chatKey === undefined ? {} : { chat: filter.chatKey }),
+        ...(filter.since === undefined ? {} : { since: filter.since }),
+        ...(filter.before === undefined ? {} : { before: filter.before }),
+    }, query, limit, (candidates) => {
+        const hashes = new Map(candidates.map(({ id, hash }) => [id, hash]));
+        const results = best(messageRows(context, accountPk, filter, { ids: [...hashes.keys()] })
+            .filter((row) => hashes.get(row.id) === row.hash)
+            .map(score));
+        return { results, best: results.map(({ id }) => id) };
+    });
+    const plain = (found) => found.map(({ chatId, messageId, score }) => ({ chatId, messageId, score }));
+    if (indexed)
+        return plain(indexed);
     const found = [];
     let after = 0;
     for (;;) {
-        const rows = orm.all(sql `SELECT k.id, ch.external_id AS chatId, m.external_id AS messageId, v.vector
-        FROM chunks k CROSS JOIN messages m ON m.id = k.chunkable_id AND m.deleted_at IS NULL
-          JOIN chats ch ON ch.id = m.chat_id
-          JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
-        WHERE k.chunkable_type = 'message' AND k.position = 0 AND k.account_id = ${accountPk} AND k.id > ${after}
-          ${chatKey === undefined ? sql `AND ch.searchable = 1` : sql `AND m.chat_id = ${chatKey}`}
-          ${since === undefined ? sql `` : sql `AND m.sent_at >= ${since}`}
-          ${before === undefined ? sql `` : sql `AND m.sent_at < ${before}`}
-        ORDER BY k.id LIMIT ${SCAN_PAGE}`);
+        const rows = messageRows(context, accountPk, filter, { after });
         for (const row of rows)
-            found.push({ chatId: row.chatId, messageId: row.messageId, score: dot(query, row.vector) });
+            found.push(score(row));
         const last = rows.at(-1);
         if (!last || rows.length < SCAN_PAGE)
             break;
         after = last.id;
     }
-    return found.sort((a, b) => b.score - a.score).slice(0, limit);
+    return plain(best(found));
 };

package/dist/store/sqlite/migrations.generated.js

@@ -739,5 +739,27 @@ export const GENERATED = [
             "INSERT OR IGNORE INTO search_index_state (name, watermark, filled_through, terms_through, normalizer_version, built_at, analyzer)\n  VALUES ('event_index', 0, 0, 0, 1, NULL, NULL);",
             "INSERT OR IGNORE INTO event_index_pending (indexable_type, id) SELECT 'event', id FROM events;"
         ]
+    },
+    {
+        "name": "20261025182500_version-34-vector-index",
+        "statements": [
+            "CREATE TABLE `vector_index_redo` (\n\t`chunk_id` integer NOT NULL\n);",
+            "CREATE TABLE `vector_indexes` (\n\t`id` integer PRIMARY KEY,\n\t`model` text NOT NULL UNIQUE,\n\t`dims` integer NOT NULL,\n\t`ready` integer NOT NULL,\n\t`filled_through` text NOT NULL\n);"
+        ]
+    },
+    {
+        "name": "20261025182501_version-34-vector-index-triggers",
+        "statements": [
+            "-- Every change that can add, drop or move a chunk's vector passes the chunk's id through vector_index_redo, where\n-- each model's own trigger (made with its vec0 table, `vector-index.ts`) re-derives that row. Writes in TypeScript\n-- and the schema's own delete triggers (document_bd, note_bd, …) all reach the index this way.\n-- A chunk usually comes before its vector; the vector's own trigger indexes it then.\n-- CROSS JOIN: led by the vectors, SQLite read every one of them per chunk (7 ms each at 100k).\nCREATE TRIGGER vector_index_chunk_ai AFTER INSERT ON chunks WHEN EXISTS (SELECT 1 FROM vector_indexes i\n    CROSS JOIN embeddings v ON v.model = i.model AND v.content_hash = new.content_hash) BEGIN\n  INSERT INTO vector_index_redo (chunk_id) VALUES (new.id);\n  DELETE FROM vector_index_redo;\nEND;",
+            "CREATE TRIGGER vector_index_chunk_ad AFTER DELETE ON chunks BEGIN\n  INSERT INTO vector_index_redo (chunk_id) VALUES (old.id);\n  DELETE FROM vector_index_redo;\nEND;",
+            "CREATE TRIGGER vector_index_chunk_au AFTER UPDATE OF content_hash, chunkable_type, chunkable_id, account_id, occurred_at\n  ON chunks WHEN old.content_hash IS NOT new.content_hash OR old.chunkable_type IS NOT new.chunkable_type\n    OR old.chunkable_id IS NOT new.chunkable_id OR old.account_id IS NOT new.account_id\n    OR old.occurred_at IS NOT new.occurred_at BEGIN\n  INSERT INTO vector_index_redo (chunk_id) VALUES (new.id);\n  DELETE FROM vector_index_redo;\nEND;",
+            "CREATE TRIGGER vector_index_embedding_ai AFTER INSERT ON embeddings\n  WHEN EXISTS (SELECT 1 FROM vector_indexes WHERE model = new.model) BEGIN\n  INSERT INTO vector_index_redo (chunk_id) SELECT id FROM chunks WHERE content_hash = new.content_hash;\n  DELETE FROM vector_index_redo;\nEND;",
+            "CREATE TRIGGER vector_index_embedding_ad AFTER DELETE ON embeddings\n  WHEN EXISTS (SELECT 1 FROM vector_indexes WHERE model = old.model) BEGIN\n  INSERT INTO vector_index_redo (chunk_id) SELECT id FROM chunks WHERE content_hash = old.content_hash;\n  DELETE FROM vector_index_redo;\nEND;",
+            "CREATE TRIGGER vector_index_embedding_au AFTER UPDATE OF model, content_hash, vector ON embeddings BEGIN\n  INSERT INTO vector_index_redo (chunk_id)\n    SELECT id FROM chunks WHERE content_hash = old.content_hash OR content_hash = new.content_hash;\n  DELETE FROM vector_index_redo;\nEND;",
+            "-- A message's chunk and its images' chunks are indexed under its chat and send time; a chat merge moves messages.\nCREATE TRIGGER vector_index_message_au AFTER UPDATE OF chat_id, sent_at ON messages\n  WHEN old.chat_id IS NOT new.chat_id OR old.sent_at IS NOT new.sent_at BEGIN\n  INSERT INTO vector_index_redo (chunk_id)\n    SELECT id FROM chunks WHERE chunkable_type = 'message' AND chunkable_id = new.id\n    UNION ALL SELECT k.id FROM attachments a JOIN chunks k ON k.chunkable_type = 'image' AND k.chunkable_id = a.id\n      WHERE a.attachable_type = 'message' AND a.attachable_id = new.id;\n  DELETE FROM vector_index_redo;\nEND;",
+            "-- An image's chat and account come from its message; a chat merge moves attachments to the kept copy.\nCREATE TRIGGER vector_index_attachment_au AFTER UPDATE OF attachable_type, attachable_id ON attachments\n  WHEN old.attachable_type IS NOT new.attachable_type OR old.attachable_id IS NOT new.attachable_id BEGIN\n  INSERT INTO vector_index_redo (chunk_id) SELECT id FROM chunks WHERE chunkable_type = 'image' AND chunkable_id = new.id;\n  DELETE FROM vector_index_redo;\nEND;",
+            "CREATE TRIGGER vector_index_conversation_au AFTER UPDATE OF chat_id ON conversations\n  WHEN old.chat_id IS NOT new.chat_id BEGIN\n  INSERT INTO vector_index_redo (chunk_id)\n    SELECT id FROM chunks WHERE chunkable_type = 'conversation' AND chunkable_id = new.id;\n  DELETE FROM vector_index_redo;\nEND;",
+            "-- Images and conversations are indexed under their chat's account; messages carry their own.\nCREATE TRIGGER vector_index_chat_au AFTER UPDATE OF account_id ON chats WHEN old.account_id IS NOT new.account_id BEGIN\n  INSERT INTO vector_index_redo (chunk_id)\n    SELECT k.id FROM messages m JOIN attachments a ON a.attachable_type = 'message' AND a.attachable_id = m.id\n      JOIN chunks k ON k.chunkable_type = 'image' AND k.chunkable_id = a.id WHERE m.chat_id = new.id\n    UNION ALL SELECT k.id FROM conversations c JOIN chunks k ON k.chunkable_type = 'conversation' AND k.chunkable_id = c.id\n      WHERE c.chat_id = new.id;\n  DELETE FROM vector_index_redo;\nEND;"
+        ]
     }
 ];

package/dist/store/sqlite/note-index.js

@@ -8,6 +8,7 @@ import { indexRow } from "./index-state.js";
 import { speech } from "./lucene-fields.js";
 import { savedStemmers } from "./stems.js";
 import { write } from "./transaction.js";
+import { dropUnusedVectors } from "./vector-index.js";
 export const CORPORA = {
     document: {
         type: "document",
@@ -114,8 +115,6 @@ const writer = (database, corpus, stemmer) => {
         stems: database.prepare(`INSERT INTO ${corpus.type}_stems (rowid, stems, scope) VALUES (?, ?, ?)`),
         chunk: database.prepare("INSERT INTO chunks (chunkable_type, chunkable_id, position, start_offset, end_offset, content_hash, scope, account_id, created_at, updated_at) " +
             "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"),
-        purge: database.prepare(`DELETE FROM embeddings WHERE content_hash = ?
-         AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ?)`),
         dequeue: database.prepare(`DELETE FROM ${corpus.type}_index_pending WHERE indexable_type = ? AND id = ?`),
     };
     return (id) => {
@@ -134,7 +133,6 @@ const writer = (database, corpus, stemmer) => {
         const stems = terms ? terms.stems : text ? stemmer.indexText(text) : "";
         if (stems !== "")
             statements.stems.run(id, stems, scope);
-        const after = new Set();
         if (text.trim() && corpus.pieces !== false) {
             const at = Date.now();
             const sections = row?.metadata == null ? [] : storedSections(String(row.metadata));
@@ -143,13 +141,10 @@ const writer = (database, corpus, stemmer) => {
                 : splitText(text, CHUNK_CHARS);
             ranges.forEach(({ start, end }, position) => {
                 const hash = chunkHash(text.slice(start, end));
-                after.add(hash);
                 statements.chunk.run(corpus.type, id, position, start, end, hash, scope, int(row?.account_id), at, at);
             });
         }
-        for (const hash of before)
-            if (!after.has(hash))
-                statements.purge.run(hash, hash);
+        dropUnusedVectors(database, before);
         if (!live && corpus.type === "document")
             dropDocumentPicture(database, id);
         statements.dequeue.run(queuedAs(corpus), id);

package/dist/store/sqlite/open-store.js

@@ -15,6 +15,7 @@ import { loadProviders } from "./providers.js";
 import { assertReadCompatible } from "./read-compatibility.js";
 import { stemsState } from "./stems.js";
 import { fillStems, followDefaultStemmers } from "./stems-fill.js";
+import { fillVectorIndexes } from "./vector-index.js";
 export const BACKFILL_ON_OPEN = 5_000;
 const NOT_WRITABLE = new Set(["EACCES", "EPERM", "EROFS"]);
 const createWritable = (file) => {
@@ -62,6 +63,9 @@ export const openStoreContext = async ({ path, env, now = Date.now, command, ver
                 stemming.watermark - stemming.filledThrough + stemming.pending <= BACKFILL_ON_OPEN)
                 fillStems(database, { now });
             logPruner(database, now)();
+            fillVectorIndexes(database, {
+                notice: (model) => warn(`indexing the stored vectors of ${model} for search by meaning — once; a large archive takes minutes`),
+            });
         }
         loadProviders(database);
         return { ...opened, now, sharedConfig: sharedConfigPath(file, env ?? process.env) };

package/dist/store/sqlite/open.js

@@ -20,10 +20,9 @@ export const openSqlite = async (path, { readOnly = false } = {}) => {
 };
 const openUnderNode = async (path, readOnly) => {
     quietSqliteWarning();
-    const { DatabaseSync } = await import("node:sqlite");
     const { drizzle } = await import("./drizzle/node.js");
-    const { cacheOverNodeSqlite } = await import("../drivers/node-sqlite.js");
-    const client = new DatabaseSync(path, { readOnly });
+    const { cacheOverNodeSqlite, nodeClient } = await import("../drivers/node-sqlite.js");
+    const client = await nodeClient(path, { readOnly });
     try {
         return { database: cacheOverNodeSqlite(client), orm: drizzle({ client }) };
     }
@@ -33,10 +32,9 @@ const openUnderNode = async (path, readOnly) => {
     }
 };
 const openUnderBun = async (path, readOnly) => {
-    const { bunDatabase, cacheOverBunSqlite } = await import("../drivers/bun-sqlite.js");
-    const Database = await bunDatabase();
+    const { bunClient, cacheOverBunSqlite } = await import("../drivers/bun-sqlite.js");
     const { drizzle } = (await import("./drizzle/bun.js"));
-    const client = readOnly ? new Database(path, { readonly: true, create: false }) : new Database(path);
+    const client = await bunClient(path, readOnly ? { readonly: true, create: false } : undefined);
     try {
         return { database: cacheOverBunSqlite(client), orm: drizzle({ client }) };
     }

package/dist/store/sqlite/schema.d.ts

@@ -15535,7 +15535,7 @@ export declare const conversationState: import("drizzle-orm/sqlite-core").SQLite
     dialect: 'sqlite';
     isAlias: false;
 }>;
-/** One model's vector for one chunk text; no owner, so vectors outlive the builds and chunks that point at them. */
+/** One model's vector for one chunk text; no owner: chunks of any kind share it, and it goes once none holds the text. */
 export declare const embeddings: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
     name: "embeddings";
     schema: undefined;
@@ -16637,6 +16637,123 @@ export declare const mentionPending: import("drizzle-orm/sqlite-core").SQLiteTab
     dialect: 'sqlite';
     isAlias: false;
 }>;
+/**
+ * One sqlite-vec `vec0` table per embedding model, `vector_index_<id>`, created at run time with its trigger since
+ * its width is the model's: a row per chunk with a vector of that model, rowid `chunks.id` (`vector-index.ts`).
+ */
+export declare const vectorIndexes: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
+    name: "vector_indexes";
+    schema: undefined;
+    columns: {
+        id: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "vector_indexes";
+            dataType: "number int53";
+            data: number;
+            driverParam: number;
+            notNull: true;
+            hasDefault: true;
+            isPrimaryKey: true;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
+        model: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "vector_indexes";
+            dataType: "string";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
+        dims: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "vector_indexes";
+            dataType: "number int53";
+            data: number;
+            driverParam: number;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
+        ready: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "vector_indexes";
+            dataType: "number int53";
+            data: number;
+            driverParam: number;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
+        filledThrough: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "vector_indexes";
+            dataType: "string";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
+    };
+    dialect: 'sqlite';
+    isAlias: false;
+}>;
+/** Always empty: each changed chunk's id passes through it so that every model's index re-derives its row. */
+export declare const vectorIndexRedo: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
+    name: "vector_index_redo";
+    schema: undefined;
+    columns: {
+        chunkId: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "vector_index_redo";
+            dataType: "number int53";
+            data: number;
+            driverParam: number;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
+    };
+    dialect: 'sqlite';
+    isAlias: false;
+}>;
 /** Settings of the store file itself, shared by every profile, tg and MAX — unlike a profile's config file. */
 export declare const storeSettings: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
     name: "store_settings";

package/dist/store/sqlite/schema.js

@@ -1556,7 +1556,7 @@ export const conversationState = sqliteTable("conversation_state", {
     /** The build readers see; a higher one is being written, or failed. */
     currentBuild: integer("current_build"),
 });
-/** One model's vector for one chunk text; no owner, so vectors outlive the builds and chunks that point at them. */
+/** One model's vector for one chunk text; no owner: chunks of any kind share it, and it goes once none holds the text. */
 export const embeddings = sqliteTable("embeddings", {
     /** `<provider>:<model>:<dims>` — vectors of different models never mix. */
     model: text("model").notNull(),
@@ -1684,6 +1684,23 @@ export const mentionPending = sqliteTable("mention_pending", {
     sourceType: text("source_type").notNull(),
     sourceId: integer("source_id").notNull(),
 }, (table) => [primaryKey({ columns: [table.sourceType, table.sourceId] })]);
+/**
+ * One sqlite-vec `vec0` table per embedding model, `vector_index_<id>`, created at run time with its trigger since
+ * its width is the model's: a row per chunk with a vector of that model, rowid `chunks.id` (`vector-index.ts`).
+ */
+export const vectorIndexes = sqliteTable("vector_indexes", {
+    id: integer("id").primaryKey(),
+    model: text("model").notNull().unique(),
+    dims: integer("dims").notNull(),
+    /** 1 once it holds every vector; until then searches scan. */
+    ready: integer("ready").notNull(),
+    /** The last `content_hash` the fill of older vectors reached. */
+    filledThrough: text("filled_through").notNull(),
+});
+/** Always empty: each changed chunk's id passes through it so that every model's index re-derives its row. */
+export const vectorIndexRedo = sqliteTable("vector_index_redo", {
+    chunkId: integer("chunk_id").notNull(),
+});
 /** Settings of the store file itself, shared by every profile, tg and MAX — unlike a profile's config file. */
 export const storeSettings = sqliteTable("store_settings", {
     key: text("key").primaryKey(),