@wirecat/cli-core 0.431.0 → 0.432.0

package/dist/store/sqlite/message-chunks.js

@@ -0,0 +1,76 @@
+import { sql } from "./drizzle/core.js";
+import { dot } from "./vectors.js";
+/** Live messages with text of the account, by pk after `after`; the caller compares hashes. */
+export const messagesToEmbed = ({ orm }, accountPk, { chatKey, model, after, limit }) => orm
+    .all(sql `SELECT m.id AS messagePk, m.text, m.sent_at AS sentAt, k.content_hash AS hash,
+          EXISTS (SELECT 1 FROM embeddings v WHERE v.model = ${model} AND v.content_hash = k.content_hash) AS embedded
+        FROM messages m
+          LEFT JOIN chunks k ON k.chunkable_type = 'message' AND k.chunkable_id = m.id AND k.position = 0
+        WHERE m.account_id = ${accountPk} AND m.deleted_at IS NULL AND trim(m.text) <> '' AND m.id > ${after}
+          ${chatKey === undefined ? sql `` : sql `AND m.chat_id = ${chatKey}`}
+        ORDER BY m.id LIMIT ${limit}`)
+    .map((row) => ({ ...row, embedded: row.embedded === 1 }));
+const dropUnshared = ({ orm }, hashes) => {
+    for (const hash of hashes)
+        orm.run(sql `DELETE FROM embeddings WHERE content_hash = ${hash}
+      AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ${hash})`);
+};
+/** Makes each hash its message's chunk; the vectors are saved apart, by `saveVectors`. */
+export const saveMessageChunks = (context, accountPk, chunks) => {
+    const { orm, now } = context;
+    const at = now();
+    const scope = orm.get(sql `SELECT scope FROM accounts WHERE id = ${accountPk}`)?.scope ?? null;
+    const old = new Set();
+    for (const { messagePk, hash, length, occurredAt } of chunks) {
+        const held = orm.get(sql `SELECT content_hash AS hash FROM chunks
+        WHERE chunkable_type = 'message' AND chunkable_id = ${messagePk} AND position = 0`);
+        if (held?.hash === hash)
+            continue;
+        if (held)
+            old.add(held.hash);
+        orm.run(sql `DELETE FROM chunks WHERE chunkable_type = 'message' AND chunkable_id = ${messagePk} AND position = 0`);
+        orm.run(sql `INSERT INTO chunks (chunkable_type, chunkable_id, position, start_offset, end_offset, content_hash, scope,
+          account_id, occurred_at, created_at, updated_at)
+        VALUES ('message', ${messagePk}, 0, 0, ${length}, ${hash}, ${scope}, ${accountPk}, ${occurredAt}, ${at}, ${at})`);
+    }
+    dropUnshared(context, old);
+};
+/**
+ * Drops the chunks of the account's messages that are gone, deleted or emptied, with the vectors nothing else
+ * uses: no trigger does it for messages, and a deleted message keeps no trace of its text.
+ */
+export const dropGoneMessageChunks = (context, accountPk) => {
+    const gone = sql `chunkable_type = 'message' AND account_id = ${accountPk} AND NOT EXISTS (SELECT 1 FROM messages m
+    WHERE m.id = chunkable_id AND m.deleted_at IS NULL AND trim(m.text) <> '')`;
+    const dropped = context.orm.all(sql `SELECT content_hash AS hash FROM chunks WHERE ${gone}`);
+    context.orm.run(sql `DELETE FROM chunks WHERE ${gone}`);
+    dropUnshared(context, dropped.map(({ hash }) => hash));
+    return dropped.length;
+};
+/** Whether any message of the account has a vector of `model`. */
+export const hasMessageVectors = ({ orm }, accountPk, model) => orm.get(sql `SELECT 1 FROM chunks k JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
+      WHERE k.chunkable_type = 'message' AND k.account_id = ${accountPk} LIMIT 1`) !== undefined;
+const SCAN_PAGE = 5_000;
+/** The account's live messages nearest to `query` by their vector of `model`, best first: a scan of every vector. */
+export const nearestMessages = ({ orm }, accountPk, { chatKey, model, since, before, limit, query, }) => {
+    const found = [];
+    let after = 0;
+    for (;;) {
+        const rows = orm.all(sql `SELECT k.id, ch.external_id AS chatId, m.external_id AS messageId, v.vector
+        FROM chunks k CROSS JOIN messages m ON m.id = k.chunkable_id AND m.deleted_at IS NULL
+          JOIN chats ch ON ch.id = m.chat_id
+          JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
+        WHERE k.chunkable_type = 'message' AND k.position = 0 AND k.account_id = ${accountPk} AND k.id > ${after}
+          ${chatKey === undefined ? sql `AND ch.searchable = 1` : sql `AND m.chat_id = ${chatKey}`}
+          ${since === undefined ? sql `` : sql `AND m.sent_at >= ${since}`}
+          ${before === undefined ? sql `` : sql `AND m.sent_at < ${before}`}
+        ORDER BY k.id LIMIT ${SCAN_PAGE}`);
+        for (const row of rows)
+            found.push({ chatId: row.chatId, messageId: row.messageId, score: dot(query, row.vector) });
+        const last = rows.at(-1);
+        if (!last || rows.length < SCAN_PAGE)
+            break;
+        after = last.id;
+    }
+    return found.sort((a, b) => b.score - a.score).slice(0, limit);
+};

package/dist/store/sqlite/vectors.js

@@ -144,7 +144,10 @@ export const clearVectors = ({ orm }, chatKey, model) => orm.all(sql `DELETE FRO
 /** Rows read per step of a search, so a chat of any size is scanned in bounded memory. */
 const SCAN_PAGE = 5_000;
 export const dot = (query, blob) => {
-    const vector = new Float32Array(blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength));
+    // A view, not a copy: copying every blob cost more than the scoring itself (cli-testing vector-index).
+    const vector = blob.byteOffset % 4 === 0
+        ? new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4)
+        : new Float32Array(blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength));
     let sum = 0;
     for (let index = 0; index < query.length; index++)
         sum += query[index] * vector[index];

package/dist/store/vectors.d.ts

@@ -1,6 +1,7 @@
 import type { Id } from "../domain/models.js";
 import { type MailStore } from "./sqlite/emails.js";
 import * as imageChunks from "./sqlite/image-chunks.js";
+import * as messageChunks from "./sqlite/message-chunks.js";
 import type { StoreContext } from "./sqlite/open.js";
 import type { ChunkToEmbed } from "./sqlite/vectors.js";
 import type { AccountKey, ConversationHit, PictureInput, StoredReadiness } from "./types.js";
@@ -51,6 +52,41 @@ export interface AccountVectors {
         messageId: Id;
         score: number;
     }[]>;
+    /** Live messages with text after pk `after`, with the chunk and vector each has; `chatId` narrows it. */
+    messagesToEmbed(key: AccountKey, options: {
+        chatId?: Id;
+        model: string;
+        after: number;
+        limit: number;
+    }): Promise<messageChunks.MessageToEmbed[]>;
+    /** Keeps each message's vector of `model` and makes its hash the message's chunk. */
+    saveMessageVectors(key: AccountKey, model: string, dims: number, messages: {
+        messagePk: number;
+        hash: string;
+        length: number;
+        occurredAt: number;
+        vector: Float32Array;
+    }[]): Promise<void>;
+    /** Drops the chunks of messages gone, deleted or emptied, and the vectors only they used; how many. */
+    dropGoneMessages(key: AccountKey): Promise<number>;
+    /** Whether any message of the account has a vector of `model`. */
+    hasMessageVectors(key: AccountKey, model: string): Promise<boolean>;
+    /**
+     * The messages nearest in meaning to `query` by their own vector of `model`, best first. The one place
+     * message vectors are read, so their storage can change behind it.
+     */
+    nearestMessages(key: AccountKey, options: {
+        chatId?: Id;
+        model: string;
+        since?: string;
+        before?: string;
+        limit: number;
+        query: Float32Array;
+    }): Promise<{
+        chatId: Id;
+        messageId: Id;
+        score: number;
+    }[]>;
     /** The current build's distinct chunks, and how many have a vector of `model`. */
     vectorStatus(key: AccountKey, chatId: Id, model: string): Promise<{
         chunks: number;

package/dist/store/vectors.js

@@ -2,6 +2,7 @@ import { keysOver } from "./keys.js";
 import * as conversationQueries from "./sqlite/conversations.js";
 import { threadConversation } from "./sqlite/emails.js";
 import * as imageChunks from "./sqlite/image-chunks.js";
+import * as messageChunks from "./sqlite/message-chunks.js";
 import { write } from "./sqlite/transaction.js";
 import * as vectors from "./sqlite/vectors.js";
 export const accountVectorsOver = (context, mail) => {
@@ -93,6 +94,59 @@ export const accountVectorsOver = (context, mail) => {
                 position: imageChunks.PICTURE_POSITION,
             });
         },
+        messagesToEmbed: async (key, { chatId, model, after, limit }) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return [];
+            const chatKey = chatId === undefined ? undefined : chatKeyOf(key, chatId);
+            if (chatId !== undefined && chatKey === undefined)
+                return [];
+            return messageChunks.messagesToEmbed(context, accountPk, {
+                ...(chatKey === undefined ? {} : { chatKey }),
+                model,
+                after,
+                limit,
+            });
+        },
+        saveMessageVectors: async (key, model, dims, messages) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return;
+            write(database, () => {
+                vectors.saveVectors(context, model, dims, messages.map(({ hash, vector }) => ({ hash, vector })));
+                messageChunks.saveMessageChunks(context, accountPk, messages);
+            });
+        },
+        dropGoneMessages: async (key) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return 0;
+            let dropped = 0;
+            write(database, () => {
+                dropped = messageChunks.dropGoneMessageChunks(context, accountPk);
+            });
+            return dropped;
+        },
+        hasMessageVectors: async (key, model) => {
+            const accountPk = findAccountPk(key);
+            return accountPk !== undefined && messageChunks.hasMessageVectors(context, accountPk, model);
+        },
+        nearestMessages: async (key, { chatId, model, since, before, limit, query }) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return [];
+            const chatKey = chatId === undefined ? undefined : chatKeyOf(key, chatId);
+            if (chatId !== undefined && chatKey === undefined)
+                return [];
+            return messageChunks.nearestMessages(context, accountPk, {
+                ...(chatKey === undefined ? {} : { chatKey }),
+                ...(since === undefined ? {} : { since: Date.parse(since) }),
+                ...(before === undefined ? {} : { before: Date.parse(before) }),
+                model,
+                limit,
+                query,
+            });
+        },
         vectorStatus: async (key, chatId, model) => {
             const chatKey = chatKeyOf(key, chatId);
             return chatKey === undefined ? { chunks: 0, embedded: 0 } : vectors.vectorStatus(context, chatKey, model);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.431.0",
+  "version": "0.432.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",