@wirecat/cli-core 0.431.0 → 0.433.0

package/dist/store/sqlite/vectors.js

@@ -144,7 +144,10 @@ export const clearVectors = ({ orm }, chatKey, model) => orm.all(sql `DELETE FRO
 /** Rows read per step of a search, so a chat of any size is scanned in bounded memory. */
 const SCAN_PAGE = 5_000;
 export const dot = (query, blob) => {
-    const vector = new Float32Array(blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength));
+    // A view, not a copy: copying every blob cost more than the scoring itself (cli-testing vector-index).
+    const vector = blob.byteOffset % 4 === 0
+        ? new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4)
+        : new Float32Array(blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength));
     let sum = 0;
     for (let index = 0; index < query.length; index++)
         sum += query[index] * vector[index];

package/dist/store/vectors.d.ts

@@ -1,6 +1,7 @@
 import type { Id } from "../domain/models.js";
 import { type MailStore } from "./sqlite/emails.js";
 import * as imageChunks from "./sqlite/image-chunks.js";
+import * as messageChunks from "./sqlite/message-chunks.js";
 import type { StoreContext } from "./sqlite/open.js";
 import type { ChunkToEmbed } from "./sqlite/vectors.js";
 import type { AccountKey, ConversationHit, PictureInput, StoredReadiness } from "./types.js";
@@ -51,6 +52,41 @@ export interface AccountVectors {
         messageId: Id;
         score: number;
     }[]>;
+    /** Live messages with text after pk `after`, with the chunk and vector each has; `chatId` narrows it. */
+    messagesToEmbed(key: AccountKey, options: {
+        chatId?: Id;
+        model: string;
+        after: number;
+        limit: number;
+    }): Promise<messageChunks.MessageToEmbed[]>;
+    /** Keeps each message's vector of `model` and makes its hash the message's chunk. */
+    saveMessageVectors(key: AccountKey, model: string, dims: number, messages: {
+        messagePk: number;
+        hash: string;
+        length: number;
+        occurredAt: number;
+        vector: Float32Array;
+    }[]): Promise<void>;
+    /** Drops the chunks of messages gone, deleted or emptied, and the vectors only they used; how many. */
+    dropGoneMessages(key: AccountKey): Promise<number>;
+    /** Whether any message of the account has a vector of `model`. */
+    hasMessageVectors(key: AccountKey, model: string): Promise<boolean>;
+    /**
+     * The messages nearest in meaning to `query` by their own vector of `model`, best first. The one place
+     * message vectors are read, so their storage can change behind it.
+     */
+    nearestMessages(key: AccountKey, options: {
+        chatId?: Id;
+        model: string;
+        since?: string;
+        before?: string;
+        limit: number;
+        query: Float32Array;
+    }): Promise<{
+        chatId: Id;
+        messageId: Id;
+        score: number;
+    }[]>;
     /** The current build's distinct chunks, and how many have a vector of `model`. */
     vectorStatus(key: AccountKey, chatId: Id, model: string): Promise<{
         chunks: number;

package/dist/store/vectors.js

@@ -2,6 +2,7 @@ import { keysOver } from "./keys.js";
 import * as conversationQueries from "./sqlite/conversations.js";
 import { threadConversation } from "./sqlite/emails.js";
 import * as imageChunks from "./sqlite/image-chunks.js";
+import * as messageChunks from "./sqlite/message-chunks.js";
 import { write } from "./sqlite/transaction.js";
 import * as vectors from "./sqlite/vectors.js";
 export const accountVectorsOver = (context, mail) => {
@@ -93,6 +94,59 @@ export const accountVectorsOver = (context, mail) => {
                 position: imageChunks.PICTURE_POSITION,
             });
         },
+        messagesToEmbed: async (key, { chatId, model, after, limit }) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return [];
+            const chatKey = chatId === undefined ? undefined : chatKeyOf(key, chatId);
+            if (chatId !== undefined && chatKey === undefined)
+                return [];
+            return messageChunks.messagesToEmbed(context, accountPk, {
+                ...(chatKey === undefined ? {} : { chatKey }),
+                model,
+                after,
+                limit,
+            });
+        },
+        saveMessageVectors: async (key, model, dims, messages) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return;
+            write(database, () => {
+                vectors.saveVectors(context, model, dims, messages.map(({ hash, vector }) => ({ hash, vector })));
+                messageChunks.saveMessageChunks(context, accountPk, messages);
+            });
+        },
+        dropGoneMessages: async (key) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return 0;
+            let dropped = 0;
+            write(database, () => {
+                dropped = messageChunks.dropGoneMessageChunks(context, accountPk);
+            });
+            return dropped;
+        },
+        hasMessageVectors: async (key, model) => {
+            const accountPk = findAccountPk(key);
+            return accountPk !== undefined && messageChunks.hasMessageVectors(context, accountPk, model);
+        },
+        nearestMessages: async (key, { chatId, model, since, before, limit, query }) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return [];
+            const chatKey = chatId === undefined ? undefined : chatKeyOf(key, chatId);
+            if (chatId !== undefined && chatKey === undefined)
+                return [];
+            return messageChunks.nearestMessages(context, accountPk, {
+                ...(chatKey === undefined ? {} : { chatKey }),
+                ...(since === undefined ? {} : { since: Date.parse(since) }),
+                ...(before === undefined ? {} : { before: Date.parse(before) }),
+                model,
+                limit,
+                query,
+            });
+        },
         vectorStatus: async (key, chatId, model) => {
             const chatKey = chatKeyOf(key, chatId);
             return chatKey === undefined ? { chunks: 0, embedded: 0 } : vectors.vectorStatus(context, chatKey, model);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.431.0",
+  "version": "0.433.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",
@@ -16,7 +16,7 @@
   "engines": {
     "node": "^22.16.0 || >=24"
   },
-  "packageManager": "pnpm@11.20.0",
+  "packageManager": "pnpm@11.28.8",
   "files": [
     "dist",
     "parity.json",
@@ -170,17 +170,17 @@
     "help:examples": "HELP_EXAMPLES_REPORT=1 vitest run scripts/help-examples.test.ts --silent=false"
   },
   "dependencies": {
-    "@anthropic-ai/sdk": "0.131.0",
+    "@anthropic-ai/sdk": "1.3.0",
     "@bomb.sh/tab": "0.0.22",
     "@huggingface/tokenizers": "0.2.0",
-    "@modelcontextprotocol/server": "2.3.0",
+    "@modelcontextprotocol/server": "^2.4.0",
     "@resvg/resvg-js": "2.6.2",
     "@valibot/to-json-schema": "1.8.0",
     "@wirecat/cli-core-onnx": "1.0.1",
     "@wirecat/cli-core-sqlite": "1.0.1",
     "@xmldom/xmldom": "0.9.12",
     "chardet": "2.2.0",
-    "defuddle": "0.19.5",
+    "defuddle": "0.19.6",
     "dictionary-en": "4.0.0",
     "dictionary-es": "4.0.0",
     "dictionary-ru": "3.0.0",
@@ -188,9 +188,9 @@
     "fflate": "0.8.3",
     "iconv-lite": "0.7.3",
     "image-size": "2.0.4",
-    "linkedom": "0.18.13",
+    "linkedom": "0.18.14",
     "liquidjs": "10.30.0",
-    "lossless-json": "4.3.0",
+    "lossless-json": "4.3.3",
     "mammoth": "1.13.0",
     "mdast-util-from-markdown": "2.1.0",
     "mdast-util-gfm": "3.1.0",
@@ -198,40 +198,39 @@
     "ogg-opus-decoder": "1.7.5",
     "psl": "1.15.0",
     "qrcode-generator": "2.0.4",
-    "sherpa-onnx": "1.13.8",
+    "sherpa-onnx": "2.0.0",
     "string-width": "8.3.0",
-    "unpdf": "1.8.1",
+    "unpdf": "1.9.1",
     "valibot": "1.5.0",
     "word-extractor": "1.0.4",
     "wrap-ansi": "10.0.2",
     "yaml": "2.9.1"
   },
   "devDependencies": {
-    "@biomejs/biome": "2.5.15",
-    "@cspell/dict-en-gb": "5.0.34",
+    "@biomejs/biome": "2.6.3",
+    "@cspell/dict-en-gb": "5.0.38",
     "@cspell/dict-es-es": "3.0.9",
-    "@cspell/dict-ru_ru": "2.3.3",
-    "@modelcontextprotocol/client": "2.3.0",
-    "@napi-rs/canvas": "1.0.10",
+    "@cspell/dict-ru_ru": "2.3.4",
+    "@modelcontextprotocol/client": "2.4.0",
+    "@napi-rs/canvas": "1.0.13",
     "@types/mdast": "4.0.4",
-    "@types/node": "^22.20.4",
+    "@types/node": "^22.20.5",
     "@types/pngjs": "6.0.5",
-    "@vitest/coverage-v8": "^5.0.2",
+    "@vitest/coverage-v8": "^5.0.5",
     "@wirecat/cli-base": "0.57.0",
     "@wirecat/cli-core-legacy-tags": "npm:@wirecat/cli-core@0.414.0",
     "commander": "15.0.0",
-    "cspell": "10.3.6",
-    "drizzle-kit": "1.0.0-rc.4",
-    "drizzle-orm": "1.0.0-rc.4",
+    "cspell": "10.4.0",
+    "drizzle-kit": "1.0.0-rc.5",
+    "drizzle-orm": "1.0.0-rc.5",
     "esbuild": "0.28.2",
-    "exceljs": "4.4.0",
     "jsqr": "1.4.0",
-    "lefthook": "2.1.16",
-    "libphonenumber-js": "1.13.14",
-    "markdownlint-cli2": "0.23.3",
+    "lefthook": "2.2.2",
+    "libphonenumber-js": "1.13.15",
+    "markdownlint-cli2": "0.24.0",
     "pngjs": "7.0.0",
     "typescript": "^7.0.2",
-    "vitest": "^5.0.1"
+    "vitest": "^5.0.5"
   },
   "publishConfig": {
     "access": "public"