@wirecat/cli-core 0.432.0 → 0.434.0

package/dist/store/sqlite/image-chunks.d.ts

@@ -46,7 +46,7 @@ export interface NearestImage {
  * The images of the account nearest to `query` by the vector of `model`, best first. Position 0 of an image is
  * its context text, position 1 its picture (`PICTURE_POSITION`): a model's vectors only ever sit at one.
  */
-export declare const nearestImages: ({ orm }: StoreContext, accountPk: number, { chatKey, model, since, limit, query, position, }: {
+export declare const nearestImages: (context: StoreContext, accountPk: number, { chatKey, model, since, limit, query, position, }: {
     chatKey?: number;
     model: string;
     since?: number;

package/dist/store/sqlite/image-chunks.js

@@ -1,4 +1,5 @@
 import { sql } from "./drizzle/core.js";
+import { searchIndex } from "./vector-index.js";
 import { dot } from "./vectors.js";
 const IS_IMAGE = sql `(a.kind = 'photo' OR a.mime LIKE 'image/%')`;
 const neighbours = ({ orm }, chatKey, messagePk, sentAt, { messages, withinMs }, side) => orm.all(sql `SELECT sender_name AS sender, text, sent_at AS sentAt FROM messages
@@ -62,32 +63,64 @@ export const embeddedHashes = ({ orm }, model, hashes) => new Set(hashes.length
               WHERE model = ${model} AND content_hash IN (SELECT value FROM json_each(${JSON.stringify(hashes)}))`)
         .map(({ hash }) => hash));
 const SCAN_PAGE = 5_000;
+/** The scan's rows: one page after `after`, or the `ids` the index picked. */
+const imageRows = ({ orm }, accountPk, { chatKey, model, since, position }, rows) => orm.all(sql `SELECT k.id, k.content_hash AS hash, ch.external_id AS chatId, m.external_id AS messageId, v.vector
+      FROM chunks k CROSS JOIN attachments a ON a.id = k.chunkable_id AND a.attachable_type = 'message'
+        JOIN messages m ON m.id = a.attachable_id AND m.deleted_at IS NULL
+        JOIN chats ch ON ch.id = m.chat_id
+        JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
+      WHERE k.chunkable_type = 'image' AND k.position = ${position} AND ch.account_id = ${accountPk}
+        ${chatKey === undefined ? sql `AND ch.searchable = 1` : sql `AND m.chat_id = ${chatKey}`}
+        ${since === undefined ? sql `` : sql `AND k.occurred_at >= ${since}`}
+        ${"ids" in rows
+    ? sql `AND k.id IN (SELECT value FROM json_each(${JSON.stringify(rows.ids)})) ORDER BY k.id`
+    : sql `AND k.id > ${rows.after} ORDER BY k.id LIMIT ${SCAN_PAGE}`}`);
 /**
  * The images of the account nearest to `query` by the vector of `model`, best first. Position 0 of an image is
  * its context text, position 1 its picture (`PICTURE_POSITION`): a model's vectors only ever sit at one.
  */
-export const nearestImages = ({ orm }, accountPk, { chatKey, model, since, limit, query, position = 0, }) => {
+export const nearestImages = (context, accountPk, { chatKey, model, since, limit, query, position = 0, }) => {
+    const filter = {
+        model,
+        position,
+        ...(chatKey === undefined ? {} : { chatKey }),
+        ...(since === undefined ? {} : { since }),
+    };
+    const score = (row) => ({
+        id: row.id,
+        chatId: row.chatId,
+        messageId: row.messageId,
+        score: dot(query, row.vector),
+    });
+    // Stable, so equal scores keep the scan's order, by chunk id.
+    const best = (found) => found.sort((a, b) => b.score - a.score).slice(0, limit);
+    const indexed = searchIndex(context.database, model, {
+        kind: "image",
+        account: accountPk,
+        ...(chatKey === undefined ? {} : { chat: chatKey }),
+        ...(since === undefined ? {} : { since }),
+    }, query, limit, (candidates) => {
+        const hashes = new Map(candidates.map(({ id, hash }) => [id, hash]));
+        const results = best(imageRows(context, accountPk, filter, { ids: [...hashes.keys()] })
+            .filter((row) => hashes.get(row.id) === row.hash)
+            .map(score));
+        return { results, best: results.map(({ id }) => id) };
+    });
+    const plain = (found) => found.map(({ chatId, messageId, score }) => ({ chatId, messageId, score }));
+    if (indexed)
+        return plain(indexed);
     const found = [];
     let after = 0;
     for (;;) {
-        const rows = orm.all(sql `SELECT k.id, ch.external_id AS chatId, m.external_id AS messageId, v.vector
-        FROM chunks k CROSS JOIN attachments a ON a.id = k.chunkable_id AND a.attachable_type = 'message'
-          JOIN messages m ON m.id = a.attachable_id AND m.deleted_at IS NULL
-          JOIN chats ch ON ch.id = m.chat_id
-          JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
-        WHERE k.chunkable_type = 'image' AND k.position = ${position} AND ch.account_id = ${accountPk}
-          AND k.id > ${after}
-          ${chatKey === undefined ? sql `AND ch.searchable = 1` : sql `AND m.chat_id = ${chatKey}`}
-          ${since === undefined ? sql `` : sql `AND k.occurred_at >= ${since}`}
-        ORDER BY k.id LIMIT ${SCAN_PAGE}`);
+        const rows = imageRows(context, accountPk, filter, { after });
         for (const row of rows)
-            found.push({ chatId: row.chatId, messageId: row.messageId, score: dot(query, row.vector) });
+            found.push(score(row));
         const last = rows.at(-1);
         if (!last || rows.length < SCAN_PAGE)
             break;
         after = last.id;
     }
-    return found.sort((a, b) => b.score - a.score).slice(0, limit);
+    return plain(best(found));
 };
 /** An image's own vector is its chunk at this position; position 0 is the text around it. */
 export const PICTURE_POSITION = 1;

package/dist/store/sqlite/manifest.js

@@ -43,6 +43,8 @@ export const MANIFEST = [
     { name: "20261025071850_version-32-files-data", version: 32, minCompatible: 32 },
     { name: "20261025072643_version-33-calendars", version: 33, minCompatible: 32 },
     { name: "20261025072644_version-33-calendars-triggers", version: 33, minCompatible: 32 },
+    { name: "20261025182500_version-34-vector-index", version: 34, minCompatible: 34 },
+    { name: "20261025182501_version-34-vector-index-triggers", version: 34, minCompatible: 34 },
 ];
 export const generatedMigrations = (manifest = MANIFEST, generated = GENERATED) => {
     const migrations = [];

package/dist/store/sqlite/message-chunks.d.ts

@@ -33,12 +33,15 @@ export interface NearestMessage {
     messageId: string;
     score: number;
 }
-/** The account's live messages nearest to `query` by their vector of `model`, best first: a scan of every vector. */
-export declare const nearestMessages: ({ orm }: StoreContext, accountPk: number, { chatKey, model, since, before, limit, query, }: {
+type MessageFilter = {
     chatKey?: number;
     model: string;
     since?: number;
     before?: number;
+};
+/** The account's live messages nearest to `query` by their vector of `model`, best first. */
+export declare const nearestMessages: (context: StoreContext, accountPk: number, { limit, query, ...filter }: MessageFilter & {
     limit: number;
     query: Float32Array;
 }) => NearestMessage[];
+export {};

package/dist/store/sqlite/message-chunks.js

@@ -1,4 +1,5 @@
 import { sql } from "./drizzle/core.js";
+import { searchIndex } from "./vector-index.js";
 import { dot } from "./vectors.js";
 /** Live messages with text of the account, by pk after `after`; the caller compares hashes. */
 export const messagesToEmbed = ({ orm }, accountPk, { chatKey, model, after, limit }) => orm
@@ -51,26 +52,54 @@ export const dropGoneMessageChunks = (context, accountPk) => {
 export const hasMessageVectors = ({ orm }, accountPk, model) => orm.get(sql `SELECT 1 FROM chunks k JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
       WHERE k.chunkable_type = 'message' AND k.account_id = ${accountPk} LIMIT 1`) !== undefined;
 const SCAN_PAGE = 5_000;
-/** The account's live messages nearest to `query` by their vector of `model`, best first: a scan of every vector. */
-export const nearestMessages = ({ orm }, accountPk, { chatKey, model, since, before, limit, query, }) => {
+/** The scan's rows: one page after `after`, or the `ids` the index picked. */
+const messageRows = ({ orm }, accountPk, { chatKey, model, since, before }, rows) => orm.all(sql `SELECT k.id, k.content_hash AS hash, ch.external_id AS chatId, m.external_id AS messageId, v.vector
+      FROM chunks k CROSS JOIN messages m ON m.id = k.chunkable_id AND m.deleted_at IS NULL
+        JOIN chats ch ON ch.id = m.chat_id
+        JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
+      WHERE k.chunkable_type = 'message' AND k.position = 0 AND k.account_id = ${accountPk}
+        ${chatKey === undefined ? sql `AND ch.searchable = 1` : sql `AND m.chat_id = ${chatKey}`}
+        ${since === undefined ? sql `` : sql `AND m.sent_at >= ${since}`}
+        ${before === undefined ? sql `` : sql `AND m.sent_at < ${before}`}
+        ${"ids" in rows
+    ? sql `AND k.id IN (SELECT value FROM json_each(${JSON.stringify(rows.ids)})) ORDER BY k.id`
+    : sql `AND k.id > ${rows.after} ORDER BY k.id LIMIT ${SCAN_PAGE}`}`);
+/** The account's live messages nearest to `query` by their vector of `model`, best first. */
+export const nearestMessages = (context, accountPk, { limit, query, ...filter }) => {
+    const score = (row) => ({
+        id: row.id,
+        chatId: row.chatId,
+        messageId: row.messageId,
+        score: dot(query, row.vector),
+    });
+    // Stable, so equal scores keep the scan's order, by chunk id.
+    const best = (found) => found.sort((a, b) => b.score - a.score).slice(0, limit);
+    const indexed = searchIndex(context.database, filter.model, {
+        kind: "message",
+        account: accountPk,
+        ...(filter.chatKey === undefined ? {} : { chat: filter.chatKey }),
+        ...(filter.since === undefined ? {} : { since: filter.since }),
+        ...(filter.before === undefined ? {} : { before: filter.before }),
+    }, query, limit, (candidates) => {
+        const hashes = new Map(candidates.map(({ id, hash }) => [id, hash]));
+        const results = best(messageRows(context, accountPk, filter, { ids: [...hashes.keys()] })
+            .filter((row) => hashes.get(row.id) === row.hash)
+            .map(score));
+        return { results, best: results.map(({ id }) => id) };
+    });
+    const plain = (found) => found.map(({ chatId, messageId, score }) => ({ chatId, messageId, score }));
+    if (indexed)
+        return plain(indexed);
     const found = [];
     let after = 0;
     for (;;) {
-        const rows = orm.all(sql `SELECT k.id, ch.external_id AS chatId, m.external_id AS messageId, v.vector
-        FROM chunks k CROSS JOIN messages m ON m.id = k.chunkable_id AND m.deleted_at IS NULL
-          JOIN chats ch ON ch.id = m.chat_id
-          JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
-        WHERE k.chunkable_type = 'message' AND k.position = 0 AND k.account_id = ${accountPk} AND k.id > ${after}
-          ${chatKey === undefined ? sql `AND ch.searchable = 1` : sql `AND m.chat_id = ${chatKey}`}
-          ${since === undefined ? sql `` : sql `AND m.sent_at >= ${since}`}
-          ${before === undefined ? sql `` : sql `AND m.sent_at < ${before}`}
-        ORDER BY k.id LIMIT ${SCAN_PAGE}`);
+        const rows = messageRows(context, accountPk, filter, { after });
         for (const row of rows)
-            found.push({ chatId: row.chatId, messageId: row.messageId, score: dot(query, row.vector) });
+            found.push(score(row));
         const last = rows.at(-1);
         if (!last || rows.length < SCAN_PAGE)
             break;
         after = last.id;
     }
-    return found.sort((a, b) => b.score - a.score).slice(0, limit);
+    return plain(best(found));
 };

package/dist/store/sqlite/migrations.generated.js

@@ -739,5 +739,27 @@ export const GENERATED = [
             "INSERT OR IGNORE INTO search_index_state (name, watermark, filled_through, terms_through, normalizer_version, built_at, analyzer)\n  VALUES ('event_index', 0, 0, 0, 1, NULL, NULL);",
             "INSERT OR IGNORE INTO event_index_pending (indexable_type, id) SELECT 'event', id FROM events;"
         ]
+    },
+    {
+        "name": "20261025182500_version-34-vector-index",
+        "statements": [
+            "CREATE TABLE `vector_index_redo` (\n\t`chunk_id` integer NOT NULL\n);",
+            "CREATE TABLE `vector_indexes` (\n\t`id` integer PRIMARY KEY,\n\t`model` text NOT NULL UNIQUE,\n\t`dims` integer NOT NULL,\n\t`ready` integer NOT NULL,\n\t`filled_through` text NOT NULL\n);"
+        ]
+    },
+    {
+        "name": "20261025182501_version-34-vector-index-triggers",
+        "statements": [
+            "-- Every change that can add, drop or move a chunk's vector passes the chunk's id through vector_index_redo, where\n-- each model's own trigger (made with its vec0 table, `vector-index.ts`) re-derives that row. Writes in TypeScript\n-- and the schema's own delete triggers (document_bd, note_bd, …) all reach the index this way.\n-- A chunk usually comes before its vector; the vector's own trigger indexes it then.\n-- CROSS JOIN: led by the vectors, SQLite read every one of them per chunk (7 ms each at 100k).\nCREATE TRIGGER vector_index_chunk_ai AFTER INSERT ON chunks WHEN EXISTS (SELECT 1 FROM vector_indexes i\n    CROSS JOIN embeddings v ON v.model = i.model AND v.content_hash = new.content_hash) BEGIN\n  INSERT INTO vector_index_redo (chunk_id) VALUES (new.id);\n  DELETE FROM vector_index_redo;\nEND;",
+            "CREATE TRIGGER vector_index_chunk_ad AFTER DELETE ON chunks BEGIN\n  INSERT INTO vector_index_redo (chunk_id) VALUES (old.id);\n  DELETE FROM vector_index_redo;\nEND;",
+            "CREATE TRIGGER vector_index_chunk_au AFTER UPDATE OF content_hash, chunkable_type, chunkable_id, account_id, occurred_at\n  ON chunks WHEN old.content_hash IS NOT new.content_hash OR old.chunkable_type IS NOT new.chunkable_type\n    OR old.chunkable_id IS NOT new.chunkable_id OR old.account_id IS NOT new.account_id\n    OR old.occurred_at IS NOT new.occurred_at BEGIN\n  INSERT INTO vector_index_redo (chunk_id) VALUES (new.id);\n  DELETE FROM vector_index_redo;\nEND;",
+            "CREATE TRIGGER vector_index_embedding_ai AFTER INSERT ON embeddings\n  WHEN EXISTS (SELECT 1 FROM vector_indexes WHERE model = new.model) BEGIN\n  INSERT INTO vector_index_redo (chunk_id) SELECT id FROM chunks WHERE content_hash = new.content_hash;\n  DELETE FROM vector_index_redo;\nEND;",
+            "CREATE TRIGGER vector_index_embedding_ad AFTER DELETE ON embeddings\n  WHEN EXISTS (SELECT 1 FROM vector_indexes WHERE model = old.model) BEGIN\n  INSERT INTO vector_index_redo (chunk_id) SELECT id FROM chunks WHERE content_hash = old.content_hash;\n  DELETE FROM vector_index_redo;\nEND;",
+            "CREATE TRIGGER vector_index_embedding_au AFTER UPDATE OF model, content_hash, vector ON embeddings BEGIN\n  INSERT INTO vector_index_redo (chunk_id)\n    SELECT id FROM chunks WHERE content_hash = old.content_hash OR content_hash = new.content_hash;\n  DELETE FROM vector_index_redo;\nEND;",
+            "-- A message's chunk and its images' chunks are indexed under its chat and send time; a chat merge moves messages.\nCREATE TRIGGER vector_index_message_au AFTER UPDATE OF chat_id, sent_at ON messages\n  WHEN old.chat_id IS NOT new.chat_id OR old.sent_at IS NOT new.sent_at BEGIN\n  INSERT INTO vector_index_redo (chunk_id)\n    SELECT id FROM chunks WHERE chunkable_type = 'message' AND chunkable_id = new.id\n    UNION ALL SELECT k.id FROM attachments a JOIN chunks k ON k.chunkable_type = 'image' AND k.chunkable_id = a.id\n      WHERE a.attachable_type = 'message' AND a.attachable_id = new.id;\n  DELETE FROM vector_index_redo;\nEND;",
+            "-- An image's chat and account come from its message; a chat merge moves attachments to the kept copy.\nCREATE TRIGGER vector_index_attachment_au AFTER UPDATE OF attachable_type, attachable_id ON attachments\n  WHEN old.attachable_type IS NOT new.attachable_type OR old.attachable_id IS NOT new.attachable_id BEGIN\n  INSERT INTO vector_index_redo (chunk_id) SELECT id FROM chunks WHERE chunkable_type = 'image' AND chunkable_id = new.id;\n  DELETE FROM vector_index_redo;\nEND;",
+            "CREATE TRIGGER vector_index_conversation_au AFTER UPDATE OF chat_id ON conversations\n  WHEN old.chat_id IS NOT new.chat_id BEGIN\n  INSERT INTO vector_index_redo (chunk_id)\n    SELECT id FROM chunks WHERE chunkable_type = 'conversation' AND chunkable_id = new.id;\n  DELETE FROM vector_index_redo;\nEND;",
+            "-- Images and conversations are indexed under their chat's account; messages carry their own.\nCREATE TRIGGER vector_index_chat_au AFTER UPDATE OF account_id ON chats WHEN old.account_id IS NOT new.account_id BEGIN\n  INSERT INTO vector_index_redo (chunk_id)\n    SELECT k.id FROM messages m JOIN attachments a ON a.attachable_type = 'message' AND a.attachable_id = m.id\n      JOIN chunks k ON k.chunkable_type = 'image' AND k.chunkable_id = a.id WHERE m.chat_id = new.id\n    UNION ALL SELECT k.id FROM conversations c JOIN chunks k ON k.chunkable_type = 'conversation' AND k.chunkable_id = c.id\n      WHERE c.chat_id = new.id;\n  DELETE FROM vector_index_redo;\nEND;"
+        ]
     }
 ];

package/dist/store/sqlite/open-store.js

@@ -15,6 +15,7 @@ import { loadProviders } from "./providers.js";
 import { assertReadCompatible } from "./read-compatibility.js";
 import { stemsState } from "./stems.js";
 import { fillStems, followDefaultStemmers } from "./stems-fill.js";
+import { fillVectorIndexes } from "./vector-index.js";
 export const BACKFILL_ON_OPEN = 5_000;
 const NOT_WRITABLE = new Set(["EACCES", "EPERM", "EROFS"]);
 const createWritable = (file) => {
@@ -62,6 +63,9 @@ export const openStoreContext = async ({ path, env, now = Date.now, command, ver
                 stemming.watermark - stemming.filledThrough + stemming.pending <= BACKFILL_ON_OPEN)
                 fillStems(database, { now });
             logPruner(database, now)();
+            fillVectorIndexes(database, {
+                notice: (model) => warn(`indexing the stored vectors of ${model} for search by meaning — once; a large archive takes minutes`),
+            });
         }
         loadProviders(database);
         return { ...opened, now, sharedConfig: sharedConfigPath(file, env ?? process.env) };

package/dist/store/sqlite/open.js

@@ -1,3 +1,4 @@
+// biome-ignore-all lint/plugin/no-sqlite-driver: cli-core owns the store, so it opens the driver itself
 import { PRAGMAS } from "../driver.js";
 import { quietSqliteWarning } from "../sqlite-warning.js";
 /**
@@ -19,10 +20,9 @@ export const openSqlite = async (path, { readOnly = false } = {}) => {
 };
 const openUnderNode = async (path, readOnly) => {
     quietSqliteWarning();
-    const { DatabaseSync } = await import("node:sqlite");
     const { drizzle } = await import("./drizzle/node.js");
-    const { cacheOverNodeSqlite } = await import("../drivers/node-sqlite.js");
-    const client = new DatabaseSync(path, { readOnly });
+    const { cacheOverNodeSqlite, nodeClient } = await import("../drivers/node-sqlite.js");
+    const client = await nodeClient(path, { readOnly });
     try {
         return { database: cacheOverNodeSqlite(client), orm: drizzle({ client }) };
     }
@@ -32,10 +32,9 @@ const openUnderNode = async (path, readOnly) => {
     }
 };
 const openUnderBun = async (path, readOnly) => {
-    const { bunDatabase, cacheOverBunSqlite } = await import("../drivers/bun-sqlite.js");
-    const Database = await bunDatabase();
+    const { bunClient, cacheOverBunSqlite } = await import("../drivers/bun-sqlite.js");
     const { drizzle } = (await import("./drizzle/bun.js"));
-    const client = readOnly ? new Database(path, { readonly: true, create: false }) : new Database(path);
+    const client = await bunClient(path, readOnly ? { readonly: true, create: false } : undefined);
     try {
         return { database: cacheOverBunSqlite(client), orm: drizzle({ client }) };
     }

package/dist/store/sqlite/schema/canonical-tags.d.ts

@@ -84,4 +84,5 @@ export declare const canonicalTags: import("drizzle-orm/sqlite-core").SQLiteTabl
         }, {}>;
     };
     dialect: 'sqlite';
+    isAlias: false;
 }>;

package/dist/store/sqlite/schema/document-index-pending.d.ts

@@ -36,4 +36,5 @@ export declare const documentIndexPending: import("drizzle-orm/sqlite-core").SQL
         }, {}>;
     };
     dialect: 'sqlite';
+    isAlias: false;
 }>;

package/dist/store/sqlite/schema/document-transfers.d.ts

@@ -100,4 +100,5 @@ export declare const documentAnnotationTransfers: import("drizzle-orm/sqlite-cor
         }, {}>;
     };
     dialect: 'sqlite';
+    isAlias: false;
 }>;

package/dist/store/sqlite/schema/tag-consolidations.d.ts

@@ -52,6 +52,7 @@ export declare const tagRedirects: import("drizzle-orm/sqlite-core").SQLiteTable
         }, {}>;
     };
     dialect: 'sqlite';
+    isAlias: false;
 }>;
 export declare const tagConsolidations: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
     name: "tag_consolidations";
@@ -267,4 +268,5 @@ export declare const tagConsolidations: import("drizzle-orm/sqlite-core").SQLite
         }, {}>;
     };
     dialect: 'sqlite';
+    isAlias: false;
 }>;