@wirecat/cli-core 0.430.0 → 0.432.0

package/dist/services/embeddings.d.ts

@@ -105,6 +105,8 @@ export interface RefreshOptions {
     maxChunks?: number;
     threads?: number;
     progress?: (note: string) => void;
+    /** With the default local model, also each message's own vector, which `--discover` reads; `false` skips it. */
+    messages?: boolean;
 }
 export interface Refreshed {
     model: string;
@@ -117,6 +119,11 @@ export interface Refreshed {
         chat: Id;
         needs: "build" | "embed";
     }[];
+    /** Messages given their own vector by this run, and how many still lack one; only when they were looked at. */
+    messages?: {
+        embedded: number;
+        left: number;
+    };
 }
 export interface FoundConversations {
     accounts?: AccountKey[];

package/dist/services/messages-archive-search.d.ts

@@ -1,6 +1,7 @@
 import type { AccountKey, MessageStore, StoredHit } from "../store/store.js";
 import type { Saved } from "./message-resolution.js";
 import type { SearchFound, SearchQuery } from "./messages.js";
+import { type MeaningLane } from "./messages-discovery.js";
 import { type MessageStats, type StatsGrouping } from "./messages-search.js";
 /** How long a search may spend building the word index, then the stems, first (phase 2 plan S3: about 200 ms). */
 export declare const SEARCH_FILL_MS = 200;
@@ -18,6 +19,6 @@ export declare const MAX_SEARCH_OFFSET = 1000;
  * Search the store, one page at a time: `offset` skips the hits a page before held, and `nextOffset` says where
  * the next page starts. A page is the hits up to `offset + limit` read again and cut, so deep pages cost more.
  */
-export declare const searchStore: (store: MessageStore, account: AccountKey, request: SearchQuery, messenger?: Saved) => Promise<SearchFound>;
+export declare const searchStore: (store: MessageStore, account: AccountKey, request: SearchQuery, messenger?: Saved, meaning?: MeaningLane) => Promise<SearchFound>;
 export declare const accountOf: (hit: StoredHit) => AccountKey;
 export declare const keyOf: ({ provider, account }: AccountKey) => string;

package/dist/services/messages-archive-search.js

@@ -43,18 +43,18 @@ export const MAX_SEARCH_OFFSET = 1000;
  * Search the store, one page at a time: `offset` skips the hits a page before held, and `nextOffset` says where
  * the next page starts. A page is the hits up to `offset + limit` read again and cut, so deep pages cost more.
  */
-export const searchStore = async (store, account, request, messenger = {}) => {
+export const searchStore = async (store, account, request, messenger = {}, meaning) => {
     const offset = request.offset ?? 0;
     if (!Number.isInteger(offset) || offset < 0 || offset > MAX_SEARCH_OFFSET)
         throw new CliError("validation_error", `offset takes a whole number from 0 to ${MAX_SEARCH_OFFSET}`);
-    const found = await searchPage(store, account, { ...request, limit: request.limit + offset }, messenger);
+    const found = await searchPage(store, account, { ...request, limit: request.limit + offset }, messenger, meaning);
     return {
         ...found,
         items: found.items.slice(offset),
         ...(found.hasMore && offset + request.limit <= MAX_SEARCH_OFFSET ? { nextOffset: offset + request.limit } : {}),
     };
 };
-const searchPage = async (store, account, request, messenger) => {
+const searchPage = async (store, account, request, messenger, meaning) => {
     if (request.signal?.aborted)
         throw new CliError("validation_error", "search was aborted", { reason: "query.aborted", complete: false });
     if (request.thread)
@@ -65,7 +65,7 @@ const searchPage = async (store, account, request, messenger) => {
         throw new CliError("validation_error", "discovery searches the local archive; use backend archive");
     await topUp(store);
     const found = request.discover
-        ? await searchDiscovery(store, account, { ...request, thread: undefined }, messenger)
+        ? await searchDiscovery(store, account, { ...request, thread: undefined }, messenger, meaning)
         : await searchLucene(store, account, request, messenger);
     return withThreads(store, found, request);
 };

package/dist/services/messages-discovery.d.ts

@@ -1,4 +1,27 @@
+import type { Id } from "../domain/models.js";
 import type { MessengerTraits } from "../ports/traits.js";
 import type { AccountKey, MessageStore } from "../store/store.js";
 import type { SearchFound, SearchQuery } from "./messages.js";
-export declare const searchDiscovery: (store: MessageStore, account: AccountKey, asked: SearchQuery, messenger?: Partial<Pick<MessengerTraits, "savedChatId" | "app" | "history" | "provider">>) => Promise<SearchFound>;
+/** How many of the messages nearest in meaning join the fusion: the benchmark's pool (performance/search/evidence-pipeline). */
+export declare const MEANING_DEPTH = 100;
+export type MeaningRanked = (AccountKey & {
+    chatId: Id;
+    id: Id;
+    score: number;
+})[];
+/** Why meaning was not searched, and the step that would make it searchable. */
+export interface MeaningSkipped {
+    reason: string;
+    /** Absent when the model itself failed: nothing the owner runs would fix it. */
+    step?: "models.download" | "conversations.rebuild";
+    model: string;
+}
+/** The messages of these accounts nearest in meaning to the question, best first; or why there are none. */
+export type MeaningLane = (question: string, accounts: AccountKey[]) => Promise<{
+    model: string;
+    ranked: MeaningRanked;
+    partly?: MeaningSkipped;
+} | {
+    skipped: MeaningSkipped;
+}>;
+export declare const searchDiscovery: (store: MessageStore, account: AccountKey, asked: SearchQuery, messenger?: Partial<Pick<MessengerTraits, "savedChatId" | "app" | "history" | "provider">>, meaning?: MeaningLane) => Promise<SearchFound>;

package/dist/services/messages-discovery.js

@@ -5,7 +5,10 @@ import { exhausted, QUERY_LIMITS } from "../search/lucene/types.js";
 import { planQuestion, words } from "../search/question-plan.js";
 import { searchCombined } from "./messages-combined.js";
 import { prepareLucene, searchLucene } from "./messages-search.js";
-export const searchDiscovery = async (store, account, asked, messenger = {}) => {
+/** How many of the messages nearest in meaning join the fusion: the benchmark's pool (performance/search/evidence-pipeline). */
+export const MEANING_DEPTH = 100;
+const RRF_K = 60;
+export const searchDiscovery = async (store, account, asked, messenger = {}, meaning) => {
     // A question's own "?" would parse as a one-character wildcard and send it to strict search (BUG-395).
     const request = asked.ast !== undefined || asked.exact || !asked.text
         ? asked
@@ -25,6 +28,8 @@ export const searchDiscovery = async (store, account, asked, messenger = {}) =>
         return searchLucene(store, account, request, messenger);
     if (terms.length > 64)
         exhausted("discovery terms");
+    // Before the clock starts: loading the model alone takes about a second of the word search's budget.
+    const lane = meaning ? await meaning(text, original.scopeAccounts) : undefined;
     const started = performance.now();
     const check = () => {
         if (request.signal?.aborted)
@@ -115,8 +120,7 @@ export const searchDiscovery = async (store, account, asked, messenger = {}) =>
     const stem = (w) => original.execution.stemmer?.stemToken(w) ?? w;
     const stems = terms.map(stem);
     let bytes = 0;
-    candidates = candidates
-        .map((m) => {
+    const described = (m) => {
         check();
         const parent = parentByLocator.get(m.locator);
         const body = `${parent ? `${parent.text}\n` : ""}${m.text}`;
@@ -137,9 +141,17 @@ export const searchDiscovery = async (store, account, asked, messenger = {}) =>
                 ...(parent ? { parent: parent.locator } : {}),
             },
         };
-    })
+    };
+    candidates = candidates
+        .map(described)
         .sort((a, b) => (b.score ?? 0) - (a.score ?? 0) ||
         Number(parentByLocator.has(b.locator)) - Number(parentByLocator.has(a.locator)));
+    let meaningCandidates = 0;
+    if (lane && "ranked" in lane) {
+        const near = await nearEligible(store, account, request, messenger, lane.ranked, plan.scope);
+        meaningCandidates = near.length;
+        candidates = fusedWithMeaning(candidates, near, described);
+    }
     check();
     const items = await Promise.all(candidates.slice(0, request.limit).map(async ({ fused: _fused, ...m }) => {
         if (!request.context)
@@ -171,7 +183,52 @@ export const searchDiscovery = async (store, account, asked, messenger = {}) =>
                 truncated,
                 terms,
                 queries: [...(plan.changed ? [] : plan.queries), relaxed],
+                ...(lane && "ranked" in lane ? { meaning: { model: lane.model, candidates: meaningCandidates } } : {}),
             },
         },
+        ...(lane && "skipped" in lane ? { meaningSkipped: lane.skipped } : {}),
+        ...(lane && "partly" in lane && lane.partly ? { meaningSkipped: lane.partly } : {}),
     };
 };
+/** The nearest messages the question's filters allow, as hits, nearest first. */
+const nearEligible = async (store, account, request, messenger, ranked, scope) => {
+    const keyOf = (one) => JSON.stringify([one.provider, one.account, one.chatId, one.id]);
+    const only = ranked
+        .slice(0, MEANING_DEPTH)
+        .filter((m) => !request.only || request.only.some((key) => keyOf(key) === keyOf(m)))
+        .map(({ provider, account, chatId, id }) => ({ provider, account, chatId, id }));
+    if (!only.length)
+        return [];
+    const eligible = await searchLucene(store, account, { ...request, text: scope || undefined, ast: undefined, limit: only.length, only, context: 0, thread: undefined }, messenger);
+    const hits = new Map(eligible.items.map((hit) => {
+        const { provider, account } = parseLocator(hit.locator);
+        return [keyOf({ provider, account, chatId: hit.chatId, id: hit.id }), hit];
+    }));
+    return ranked.flatMap((m) => {
+        const hit = hits.get(keyOf(m));
+        return hit ? [{ ...hit, cosine: m.score }] : [];
+    });
+};
+/**
+ * Reciprocal rank fusion of discovery's order and the meaning order, k = 60, as the benchmark's A2 arm: a hit
+ * scores 1 / (k + rank) in each list it is in. Coverage is left out of the fused score, so a message that answers
+ * in other words can rise above one that only repeats them.
+ */
+const fusedWithMeaning = (words, near, described) => {
+    const merged = new Map();
+    const by = (hit, foundBy) => hit.discovery ? { ...hit, discovery: { ...hit.discovery, foundBy } } : hit;
+    words.forEach((hit, rank) => {
+        merged.set(hit.locator, { hit: by(hit, ["words"]), rrf: 1 / (RRF_K + rank + 1) });
+    });
+    near.forEach(({ cosine: _, ...m }, rank) => {
+        const share = 1 / (RRF_K + rank + 1);
+        const held = merged.get(m.locator);
+        if (held) {
+            held.rrf += share;
+            held.hit = by(held.hit, ["words", "meaning"]);
+        }
+        else
+            merged.set(m.locator, { hit: by(described({ ...m, fused: 0 }), ["meaning"]), rrf: share });
+    });
+    return [...merged.values()].sort((a, b) => b.rrf - a.rrf).map(({ hit, rrf }) => ({ ...hit, score: rrf }));
+};

package/dist/services/messages-meaning.d.ts

@@ -0,0 +1,7 @@
+import type { ServiceDeps } from "./deps.js";
+import { type MeaningLane } from "./messages-discovery.js";
+/**
+ * Discovery's meaning half: the default local model over each message's own vector. It runs only when the model is
+ * on disk and vectors exist — never a download, never a remote model — and any failure leaves the words alone.
+ */
+export declare const meaningLane: (deps: ServiceDeps) => MeaningLane;

package/dist/services/messages-meaning.js

@@ -0,0 +1,53 @@
+import { DEFAULT_TEXT_MODEL } from "../embeddings/models.js";
+import { embeddingTarget, withEmbedder } from "./embeddings-maintenance.js";
+import { MEANING_DEPTH } from "./messages-discovery.js";
+import { plainWords } from "./pictures.js";
+/**
+ * Discovery's meaning half: the default local model over each message's own vector. It runs only when the model is
+ * on disk and vectors exist — never a download, never a remote model — and any failure leaves the words alone.
+ */
+export const meaningLane = (deps) => async (question, accounts) => {
+    const target = embeddingTarget(deps, DEFAULT_TEXT_MODEL);
+    const skipped = (reason, step) => ({
+        skipped: { reason, model: target.id, ...(step ? { step } : {}) },
+    });
+    if (!target.installed)
+        return skipped(`${target.id} is not downloaded`, "models.download");
+    const store = await deps.store();
+    const ready = [];
+    const without = [];
+    for (const one of accounts)
+        ((await store.vectors.hasMessageVectors(one, target.key)) ? ready : without).push(one);
+    if (ready.length === 0)
+        return skipped("no message has a vector yet", "conversations.rebuild");
+    const words = plainWords(question);
+    try {
+        const ranked = [];
+        if (words) {
+            await withEmbedder(deps, target, DEFAULT_TEXT_MODEL, {}, async (embedder) => {
+                const [query] = await embedder.embed([words], "query");
+                for (const one of ready) {
+                    const near = await store.vectors.nearestMessages(one, {
+                        model: target.key,
+                        limit: MEANING_DEPTH,
+                        query: query,
+                    });
+                    for (const { chatId, messageId, score } of near)
+                        ranked.push({ ...one, chatId, id: messageId, score });
+                }
+            });
+        }
+        return {
+            model: target.id,
+            ranked: ranked.sort((a, b) => b.score - a.score).slice(0, MEANING_DEPTH),
+            ...(without.length
+                ? {
+                    partly: skipped(`${without.map(({ provider, account }) => `${provider}:${account}`).join(", ")} has no message vectors yet`, "conversations.rebuild").skipped,
+                }
+                : {}),
+        };
+    }
+    catch (error) {
+        return skipped(`${target.id} failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+};

package/dist/services/messages-search.d.ts

@@ -15,6 +15,11 @@ export interface QueryMetadata {
         truncated: boolean;
         terms: string[];
         queries: string[];
+        /** Present when meaning was searched too: the model, and how many of its nearest messages the filters allowed. */
+        meaning?: {
+            model: string;
+            candidates: number;
+        };
     };
     version: 1;
     fieldsVersion: number;

package/dist/services/messages.d.ts

@@ -7,6 +7,7 @@ import { type Operated } from "../sends/guarded.js";
 import type { PermissionKey } from "../sends/permissions.js";
 import type { AccountKey, ChatCompleteness, MatchPlace, StoredHit } from "../store/store.js";
 import { type ServiceDeps } from "./deps.js";
+import type { MeaningSkipped } from "./messages-discovery.js";
 import type { MessageStats, QueryMetadata, SearchCoverage, StatsGrouping } from "./messages-search.js";
 import type { PictureHit } from "./pictures.js";
 import { type SearchAllFound, type SearchAllRequest } from "./search-all.js";
@@ -79,6 +80,8 @@ export type FoundMessage = StoredHit & {
         matchedTerms: string[];
         missingTerms: string[];
         parent?: string;
+        /** Only when meaning was searched too: which of the two lists found it. */
+        foundBy?: ("words" | "meaning")[];
     };
     match?: "words" | "beginnings" | "corrected";
     /** The garbled spellings a `corrected` hit's transcript holds, per query word. */
@@ -116,6 +119,8 @@ export interface SearchFound extends Page<FoundMessage> {
     /** Per chat of the page: how much of its history the store holds. */
     completeness: (ChatCompleteness & AccountKey)[];
     /** `false` while the word index is being built. */
+    /** `--discover` searched by words alone: why meaning was not searched, and the step that would enable it. */
+    meaningSkipped?: MeaningSkipped;
     wordsReady: boolean;
     /** The same for the stems, in strict search: `false` until a stemmed search may run. */
     stemsReady?: boolean;

package/dist/services/messages.js

@@ -12,6 +12,7 @@ import { uploadSize } from "../sends/upload.js";
 import { crossAccount, fromStore, PUSHED } from "./deps.js";
 import { readChatId, storedChatId } from "./message-resolution.js";
 import { accountOf, keyOf, searchStore, statsStore, topUp } from "./messages-archive-search.js";
+import { meaningLane } from "./messages-meaning.js";
 import { searchAll } from "./search-all.js";
 import { searchAllWithMeetings, } from "./search-all-meetings.js";
 import { searchRecordOf } from "./search-record.js";
@@ -31,7 +32,7 @@ export const messagesService = (deps) => {
                 return { ...account, chatId, id };
             })
             : undefined;
-        const found = await searchStore(store, account, only ? { ...query, only } : query, deps.messenger);
+        const found = await searchStore(store, account, only ? { ...query, only } : query, deps.messenger, query.discover && query.kind !== "mail" ? meaningLane(deps) : undefined);
         return { found, server };
     };
     const unifiedMessages = (store, account) => async (query) => {

package/dist/services/search-catchup.js

@@ -36,6 +36,7 @@ export const catchUpSearch = async (deps, chat, options, signal, clock = Date.no
             maxMessages,
             check,
             threads: 1,
+            messages: false,
         });
         check();
         return {

package/dist/store/sqlite/message-chunks.d.ts

@@ -0,0 +1,44 @@
+import type { StoreContext } from "./open.js";
+/** A live message with text, and the chunk and vector it already has, if any. */
+export interface MessageToEmbed {
+    messagePk: number;
+    text: string;
+    sentAt: number;
+    hash: string | null;
+    embedded: boolean;
+}
+/** Live messages with text of the account, by pk after `after`; the caller compares hashes. */
+export declare const messagesToEmbed: ({ orm }: StoreContext, accountPk: number, { chatKey, model, after, limit }: {
+    chatKey?: number;
+    model: string;
+    after: number;
+    limit: number;
+}) => MessageToEmbed[];
+/** Makes each hash its message's chunk; the vectors are saved apart, by `saveVectors`. */
+export declare const saveMessageChunks: (context: StoreContext, accountPk: number, chunks: {
+    messagePk: number;
+    hash: string;
+    length: number;
+    occurredAt: number;
+}[]) => void;
+/**
+ * Drops the chunks of the account's messages that are gone, deleted or emptied, with the vectors nothing else
+ * uses: no trigger does it for messages, and a deleted message keeps no trace of its text.
+ */
+export declare const dropGoneMessageChunks: (context: StoreContext, accountPk: number) => number;
+/** Whether any message of the account has a vector of `model`. */
+export declare const hasMessageVectors: ({ orm }: StoreContext, accountPk: number, model: string) => boolean;
+export interface NearestMessage {
+    chatId: string;
+    messageId: string;
+    score: number;
+}
+/** The account's live messages nearest to `query` by their vector of `model`, best first: a scan of every vector. */
+export declare const nearestMessages: ({ orm }: StoreContext, accountPk: number, { chatKey, model, since, before, limit, query, }: {
+    chatKey?: number;
+    model: string;
+    since?: number;
+    before?: number;
+    limit: number;
+    query: Float32Array;
+}) => NearestMessage[];

package/dist/store/sqlite/message-chunks.js

@@ -0,0 +1,76 @@
+import { sql } from "./drizzle/core.js";
+import { dot } from "./vectors.js";
+/** Live messages with text of the account, by pk after `after`; the caller compares hashes. */
+export const messagesToEmbed = ({ orm }, accountPk, { chatKey, model, after, limit }) => orm
+    .all(sql `SELECT m.id AS messagePk, m.text, m.sent_at AS sentAt, k.content_hash AS hash,
+          EXISTS (SELECT 1 FROM embeddings v WHERE v.model = ${model} AND v.content_hash = k.content_hash) AS embedded
+        FROM messages m
+          LEFT JOIN chunks k ON k.chunkable_type = 'message' AND k.chunkable_id = m.id AND k.position = 0
+        WHERE m.account_id = ${accountPk} AND m.deleted_at IS NULL AND trim(m.text) <> '' AND m.id > ${after}
+          ${chatKey === undefined ? sql `` : sql `AND m.chat_id = ${chatKey}`}
+        ORDER BY m.id LIMIT ${limit}`)
+    .map((row) => ({ ...row, embedded: row.embedded === 1 }));
+const dropUnshared = ({ orm }, hashes) => {
+    for (const hash of hashes)
+        orm.run(sql `DELETE FROM embeddings WHERE content_hash = ${hash}
+      AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ${hash})`);
+};
+/** Makes each hash its message's chunk; the vectors are saved apart, by `saveVectors`. */
+export const saveMessageChunks = (context, accountPk, chunks) => {
+    const { orm, now } = context;
+    const at = now();
+    const scope = orm.get(sql `SELECT scope FROM accounts WHERE id = ${accountPk}`)?.scope ?? null;
+    const old = new Set();
+    for (const { messagePk, hash, length, occurredAt } of chunks) {
+        const held = orm.get(sql `SELECT content_hash AS hash FROM chunks
+        WHERE chunkable_type = 'message' AND chunkable_id = ${messagePk} AND position = 0`);
+        if (held?.hash === hash)
+            continue;
+        if (held)
+            old.add(held.hash);
+        orm.run(sql `DELETE FROM chunks WHERE chunkable_type = 'message' AND chunkable_id = ${messagePk} AND position = 0`);
+        orm.run(sql `INSERT INTO chunks (chunkable_type, chunkable_id, position, start_offset, end_offset, content_hash, scope,
+          account_id, occurred_at, created_at, updated_at)
+        VALUES ('message', ${messagePk}, 0, 0, ${length}, ${hash}, ${scope}, ${accountPk}, ${occurredAt}, ${at}, ${at})`);
+    }
+    dropUnshared(context, old);
+};
+/**
+ * Drops the chunks of the account's messages that are gone, deleted or emptied, with the vectors nothing else
+ * uses: no trigger does it for messages, and a deleted message keeps no trace of its text.
+ */
+export const dropGoneMessageChunks = (context, accountPk) => {
+    const gone = sql `chunkable_type = 'message' AND account_id = ${accountPk} AND NOT EXISTS (SELECT 1 FROM messages m
+    WHERE m.id = chunkable_id AND m.deleted_at IS NULL AND trim(m.text) <> '')`;
+    const dropped = context.orm.all(sql `SELECT content_hash AS hash FROM chunks WHERE ${gone}`);
+    context.orm.run(sql `DELETE FROM chunks WHERE ${gone}`);
+    dropUnshared(context, dropped.map(({ hash }) => hash));
+    return dropped.length;
+};
+/** Whether any message of the account has a vector of `model`. */
+export const hasMessageVectors = ({ orm }, accountPk, model) => orm.get(sql `SELECT 1 FROM chunks k JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
+      WHERE k.chunkable_type = 'message' AND k.account_id = ${accountPk} LIMIT 1`) !== undefined;
+const SCAN_PAGE = 5_000;
+/** The account's live messages nearest to `query` by their vector of `model`, best first: a scan of every vector. */
+export const nearestMessages = ({ orm }, accountPk, { chatKey, model, since, before, limit, query, }) => {
+    const found = [];
+    let after = 0;
+    for (;;) {
+        const rows = orm.all(sql `SELECT k.id, ch.external_id AS chatId, m.external_id AS messageId, v.vector
+        FROM chunks k CROSS JOIN messages m ON m.id = k.chunkable_id AND m.deleted_at IS NULL
+          JOIN chats ch ON ch.id = m.chat_id
+          JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
+        WHERE k.chunkable_type = 'message' AND k.position = 0 AND k.account_id = ${accountPk} AND k.id > ${after}
+          ${chatKey === undefined ? sql `AND ch.searchable = 1` : sql `AND m.chat_id = ${chatKey}`}
+          ${since === undefined ? sql `` : sql `AND m.sent_at >= ${since}`}
+          ${before === undefined ? sql `` : sql `AND m.sent_at < ${before}`}
+        ORDER BY k.id LIMIT ${SCAN_PAGE}`);
+        for (const row of rows)
+            found.push({ chatId: row.chatId, messageId: row.messageId, score: dot(query, row.vector) });
+        const last = rows.at(-1);
+        if (!last || rows.length < SCAN_PAGE)
+            break;
+        after = last.id;
+    }
+    return found.sort((a, b) => b.score - a.score).slice(0, limit);
+};

package/dist/store/sqlite/vectors.js

@@ -144,7 +144,10 @@ export const clearVectors = ({ orm }, chatKey, model) => orm.all(sql `DELETE FRO
 /** Rows read per step of a search, so a chat of any size is scanned in bounded memory. */
 const SCAN_PAGE = 5_000;
 export const dot = (query, blob) => {
-    const vector = new Float32Array(blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength));
+    // A view, not a copy: copying every blob cost more than the scoring itself (cli-testing vector-index).
+    const vector = blob.byteOffset % 4 === 0
+        ? new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4)
+        : new Float32Array(blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength));
     let sum = 0;
     for (let index = 0; index < query.length; index++)
         sum += query[index] * vector[index];

package/dist/store/vectors.d.ts

@@ -1,6 +1,7 @@
 import type { Id } from "../domain/models.js";
 import { type MailStore } from "./sqlite/emails.js";
 import * as imageChunks from "./sqlite/image-chunks.js";
+import * as messageChunks from "./sqlite/message-chunks.js";
 import type { StoreContext } from "./sqlite/open.js";
 import type { ChunkToEmbed } from "./sqlite/vectors.js";
 import type { AccountKey, ConversationHit, PictureInput, StoredReadiness } from "./types.js";
@@ -51,6 +52,41 @@ export interface AccountVectors {
         messageId: Id;
         score: number;
     }[]>;
+    /** Live messages with text after pk `after`, with the chunk and vector each has; `chatId` narrows it. */
+    messagesToEmbed(key: AccountKey, options: {
+        chatId?: Id;
+        model: string;
+        after: number;
+        limit: number;
+    }): Promise<messageChunks.MessageToEmbed[]>;
+    /** Keeps each message's vector of `model` and makes its hash the message's chunk. */
+    saveMessageVectors(key: AccountKey, model: string, dims: number, messages: {
+        messagePk: number;
+        hash: string;
+        length: number;
+        occurredAt: number;
+        vector: Float32Array;
+    }[]): Promise<void>;
+    /** Drops the chunks of messages gone, deleted or emptied, and the vectors only they used; how many. */
+    dropGoneMessages(key: AccountKey): Promise<number>;
+    /** Whether any message of the account has a vector of `model`. */
+    hasMessageVectors(key: AccountKey, model: string): Promise<boolean>;
+    /**
+     * The messages nearest in meaning to `query` by their own vector of `model`, best first. The one place
+     * message vectors are read, so their storage can change behind it.
+     */
+    nearestMessages(key: AccountKey, options: {
+        chatId?: Id;
+        model: string;
+        since?: string;
+        before?: string;
+        limit: number;
+        query: Float32Array;
+    }): Promise<{
+        chatId: Id;
+        messageId: Id;
+        score: number;
+    }[]>;
     /** The current build's distinct chunks, and how many have a vector of `model`. */
     vectorStatus(key: AccountKey, chatId: Id, model: string): Promise<{
         chunks: number;

package/dist/store/vectors.js

@@ -2,6 +2,7 @@ import { keysOver } from "./keys.js";
 import * as conversationQueries from "./sqlite/conversations.js";
 import { threadConversation } from "./sqlite/emails.js";
 import * as imageChunks from "./sqlite/image-chunks.js";
+import * as messageChunks from "./sqlite/message-chunks.js";
 import { write } from "./sqlite/transaction.js";
 import * as vectors from "./sqlite/vectors.js";
 export const accountVectorsOver = (context, mail) => {
@@ -93,6 +94,59 @@ export const accountVectorsOver = (context, mail) => {
                 position: imageChunks.PICTURE_POSITION,
             });
         },
+        messagesToEmbed: async (key, { chatId, model, after, limit }) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return [];
+            const chatKey = chatId === undefined ? undefined : chatKeyOf(key, chatId);
+            if (chatId !== undefined && chatKey === undefined)
+                return [];
+            return messageChunks.messagesToEmbed(context, accountPk, {
+                ...(chatKey === undefined ? {} : { chatKey }),
+                model,
+                after,
+                limit,
+            });
+        },
+        saveMessageVectors: async (key, model, dims, messages) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return;
+            write(database, () => {
+                vectors.saveVectors(context, model, dims, messages.map(({ hash, vector }) => ({ hash, vector })));
+                messageChunks.saveMessageChunks(context, accountPk, messages);
+            });
+        },
+        dropGoneMessages: async (key) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return 0;
+            let dropped = 0;
+            write(database, () => {
+                dropped = messageChunks.dropGoneMessageChunks(context, accountPk);
+            });
+            return dropped;
+        },
+        hasMessageVectors: async (key, model) => {
+            const accountPk = findAccountPk(key);
+            return accountPk !== undefined && messageChunks.hasMessageVectors(context, accountPk, model);
+        },
+        nearestMessages: async (key, { chatId, model, since, before, limit, query }) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return [];
+            const chatKey = chatId === undefined ? undefined : chatKeyOf(key, chatId);
+            if (chatId !== undefined && chatKey === undefined)
+                return [];
+            return messageChunks.nearestMessages(context, accountPk, {
+                ...(chatKey === undefined ? {} : { chatKey }),
+                ...(since === undefined ? {} : { since: Date.parse(since) }),
+                ...(before === undefined ? {} : { before: Date.parse(before) }),
+                model,
+                limit,
+                query,
+            });
+        },
         vectorStatus: async (key, chatId, model) => {
             const chatKey = chatKeyOf(key, chatId);
             return chatKey === undefined ? { chunks: 0, embedded: 0 } : vectors.vectorStatus(context, chatKey, model);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.430.0",
+  "version": "0.432.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",