@wirecat/cli-core 0.431.0 → 0.432.0

package/dist/entities/message-transfer.d.ts

@@ -184,6 +184,10 @@ export declare const messageSyncChat: v.StrictObjectSchema<{
                 readonly chat: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 1, undefined>]>;
                 readonly needs: v.PicklistSchema<["build", "embed"], undefined>;
             }, undefined>, undefined>;
+            readonly messages: v.OptionalSchema<v.StrictObjectSchema<{
+                readonly embedded: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
+                readonly left: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
+            }, undefined>, undefined>;
         }, undefined>, undefined>;
     }, undefined>, undefined>;
     readonly window_complete: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
@@ -304,6 +308,10 @@ export declare const messageSync: v.UnionSchema<[v.StrictObjectSchema<{
                 readonly chat: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 1, undefined>]>;
                 readonly needs: v.PicklistSchema<["build", "embed"], undefined>;
             }, undefined>, undefined>;
+            readonly messages: v.OptionalSchema<v.StrictObjectSchema<{
+                readonly embedded: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
+                readonly left: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
+            }, undefined>, undefined>;
         }, undefined>, undefined>;
     }, undefined>, undefined>;
     readonly window_complete: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;

package/dist/entities/message-write.d.ts

@@ -32,6 +32,7 @@ export declare const messageWrite: v.StrictObjectSchema<{
             readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
             readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
             readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
+            readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
         }, undefined>, undefined>;
         readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
             readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;
@@ -219,6 +220,7 @@ export declare const messageWrite: v.StrictObjectSchema<{
             readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
             readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
             readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
+            readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
         }, undefined>, undefined>;
         readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
             readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;
@@ -414,6 +416,7 @@ export declare const messageSend: v.UnionSchema<[v.StrictObjectSchema<{
             readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
             readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
             readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
+            readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
         }, undefined>, undefined>;
         readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
             readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;
@@ -601,6 +604,7 @@ export declare const messageSend: v.UnionSchema<[v.StrictObjectSchema<{
             readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
             readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
             readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
+            readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
         }, undefined>, undefined>;
         readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
             readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;

package/dist/entities/message.d.ts

@@ -42,6 +42,8 @@ export declare const message: v.StrictObjectSchema<{
         readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
         readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
         readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
+        /** Present when meaning was searched too: which of the two lists found it. */
+        readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
     }, undefined>, undefined>;
     /** The messages around a search hit, or its thread, oldest first; the hit itself is among them. */
     readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{

package/dist/entities/message.js

@@ -51,6 +51,8 @@ export const message = v.strictObject({
         matched_terms: v.array(text),
         missing_terms: v.array(text),
         parent: v.optional(reference),
+        /** Present when meaning was searched too: which of the two lists found it. */
+        found_by: v.optional(v.array(v.picklist(["words", "meaning"]))),
     })),
     /** The messages around a search hit, or its thread, oldest first; the hit itself is among them. */
     context: v.optional(v.array(v.strictObject({ ref: reference, timestamp: time, sender_name: v.nullable(text), text }))),

package/dist/entities/store-records.d.ts

@@ -55,6 +55,7 @@ export declare const storeExport: v.UnionSchema<[v.StrictObjectSchema<{
         readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
         readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
         readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
+        readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
     }, undefined>, undefined>;
     readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
         readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;
@@ -352,6 +353,10 @@ export declare const storeGapRepair: v.UnionSchema<[v.StrictObjectSchema<{
                 readonly chat: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 1, undefined>]>;
                 readonly needs: v.PicklistSchema<["build", "embed"], undefined>;
             }, undefined>, undefined>;
+            readonly messages: v.OptionalSchema<v.StrictObjectSchema<{
+                readonly embedded: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
+                readonly left: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
+            }, undefined>, undefined>;
         }, undefined>, undefined>;
     }, undefined>, undefined>;
     readonly stopped: v.OptionalSchema<v.PicklistSchema<["bound", "partial_or_inaccessible", "fetch_failed"], undefined>, undefined>;

package/dist/services/embeddings-maintenance.js

@@ -11,6 +11,8 @@ import { conversationsService } from "./conversations.js";
 import { storedChatId } from "./message-resolution.js";
 /** How much one run without `--chat` does at most, by default. */
 export const REFRESH_BOUNDS = { maxChats: 20, maxChunks: 2_000 };
+/** Messages read per page while looking for those without a current vector. */
+const MESSAGE_PAGE = 5_000;
 /** Chunks per batch for one local session; one short transaction writes each batch. */
 const PER_SESSION = 8;
 /** Texts per remote request; `concurrency` of them run at once. */
@@ -169,6 +171,58 @@ export const embeddingsMaintenance = (deps) => {
         })), described.map(({ attachmentPk }) => attachmentPk));
         return missing.length;
     };
+    /**
+     * The messages of the account, or of one chat, whose own vector is missing or older than their text: at most
+     * `budget` of them in hand, and how many more there are. Gone messages lose their chunks first.
+     */
+    const messagesToEmbed = async (chatId, model, budget, check) => {
+        const store = await deps.store();
+        const account = await deps.account();
+        await store.vectors.dropGoneMessages(account);
+        const items = [];
+        let beyond = 0;
+        let after = 0;
+        for (;;) {
+            check?.();
+            const page = await store.vectors.messagesToEmbed(account, {
+                ...(chatId === undefined ? {} : { chatId }),
+                model,
+                after,
+                limit: MESSAGE_PAGE,
+            });
+            const last = page.at(-1);
+            if (!last)
+                break;
+            after = last.messagePk;
+            for (const { messagePk, text, sentAt, hash: held, embedded } of page) {
+                const hash = chunkHash(text);
+                if (hash === held && embedded)
+                    continue;
+                if (items.length < budget)
+                    items.push({ messagePk, text, hash, occurredAt: sentAt });
+                else
+                    beyond++;
+            }
+        }
+        return { items, beyond };
+    };
+    const embedMessages = async (items, target, embedder, check) => {
+        const store = await deps.store();
+        const account = await deps.account();
+        for (let at = 0; at < items.length; at += target.perBatch) {
+            check?.();
+            const batch = items.slice(at, at + target.perBatch);
+            const vectors = await embedder.embed(batch.map(({ text }) => text), "passage");
+            await store.vectors.saveMessageVectors(account, target.key, target.dims, batch.map(({ messagePk, hash, text, occurredAt }, index) => ({
+                messagePk,
+                hash,
+                length: text.length,
+                occurredAt,
+                vector: vectors[index],
+            })));
+        }
+        return items.length;
+    };
     return {
         status: async (chat, choice) => {
             const { store, account, chatId } = await found(chat);
@@ -204,7 +258,7 @@ export const embeddingsMaintenance = (deps) => {
                 return { model: target.id, chats };
             return { model: target.id, chats, unbuiltGroups: (await store.vectors.unbuiltGroups(account)).length };
         },
-        refresh: async ({ chat, model: choice = DEFAULT_TEXT_MODEL, build = true, embed = true, maxChats = REFRESH_BOUNDS.maxChats, maxChunks = REFRESH_BOUNDS.maxChunks, threads, progress, maxMessages, check, }) => {
+        refresh: async ({ chat, model: choice = DEFAULT_TEXT_MODEL, build = true, embed = true, maxChats = REFRESH_BOUNDS.maxChats, maxChunks = REFRESH_BOUNDS.maxChunks, threads, progress, maxMessages, check, messages = true, }) => {
             check?.();
             const store = await deps.store();
             const account = await deps.account();
@@ -236,13 +290,16 @@ export const embeddingsMaintenance = (deps) => {
                 }
             }
             const embedded = [];
+            const wantsMessages = embed && messages && target.installed && choice === DEFAULT_TEXT_MODEL;
+            const toEmbed = wantsMessages ? await messagesToEmbed(chatId, target.key, maxChunks, check) : undefined;
+            let messagesEmbedded = 0;
             if (embed && target.installed) {
                 const fresh = new Set(built.map(({ chat }) => chat));
                 const queue = (await scan()).chats
                     .filter(needsEmbed)
                     .sort((a, b) => Number(fresh.has(b.chat)) - Number(fresh.has(a.chat)))
                     .slice(0, maxChats);
-                if (queue.length > 0) {
+                if (queue.length > 0 || toEmbed?.items.length) {
                     await withEmbedder(deps, target, choice, { threads }, async (embedder) => {
                         let budget = maxChunks;
                         for (const { chat } of queue) {
@@ -254,6 +311,10 @@ export const embeddingsMaintenance = (deps) => {
                             embedded.push(done);
                             progress?.(`chat ${chat}: ${done.embedded} chunks embedded with ${target.id}`);
                         }
+                        if (toEmbed?.items.length && budget > 0) {
+                            messagesEmbedded = await embedMessages(toEmbed.items.slice(0, budget), target, embedder, check);
+                            progress?.(`${messagesEmbedded} messages embedded with ${target.id}`);
+                        }
                     });
                 }
             }
@@ -273,6 +334,14 @@ export const embeddingsMaintenance = (deps) => {
                             .map(({ chat }) => ({ chat, needs: "embed" }))
                         : []),
                 ],
+                ...(toEmbed
+                    ? {
+                        messages: {
+                            embedded: messagesEmbedded,
+                            left: toEmbed.items.length + toEmbed.beyond - messagesEmbedded,
+                        },
+                    }
+                    : {}),
             };
         },
         clear: async (chat, choice) => {

package/dist/services/embeddings-search.js

@@ -122,7 +122,9 @@ export const embeddingsSearch = (deps) => {
                 source,
                 timezone,
             });
-            const locallyRefreshed = refresh ? await embeddingsMaintenance(deps).refresh(refresh) : undefined;
+            const locallyRefreshed = refresh
+                ? await embeddingsMaintenance(deps).refresh({ ...refresh, messages: false })
+                : undefined;
             const store = await deps.store();
             const account = await deps.account();
             const target = embeddingTarget(deps, choice);

package/dist/services/embeddings.d.ts

@@ -105,6 +105,8 @@ export interface RefreshOptions {
     maxChunks?: number;
     threads?: number;
     progress?: (note: string) => void;
+    /** With the default local model, also each message's own vector, which `--discover` reads; `false` skips it. */
+    messages?: boolean;
 }
 export interface Refreshed {
     model: string;
@@ -117,6 +119,11 @@ export interface Refreshed {
         chat: Id;
         needs: "build" | "embed";
     }[];
+    /** Messages given their own vector by this run, and how many still lack one; only when they were looked at. */
+    messages?: {
+        embedded: number;
+        left: number;
+    };
 }
 export interface FoundConversations {
     accounts?: AccountKey[];

package/dist/services/messages-archive-search.d.ts

@@ -1,6 +1,7 @@
 import type { AccountKey, MessageStore, StoredHit } from "../store/store.js";
 import type { Saved } from "./message-resolution.js";
 import type { SearchFound, SearchQuery } from "./messages.js";
+import { type MeaningLane } from "./messages-discovery.js";
 import { type MessageStats, type StatsGrouping } from "./messages-search.js";
 /** How long a search may spend building the word index, then the stems, first (phase 2 plan S3: about 200 ms). */
 export declare const SEARCH_FILL_MS = 200;
@@ -18,6 +19,6 @@ export declare const MAX_SEARCH_OFFSET = 1000;
  * Search the store, one page at a time: `offset` skips the hits a page before held, and `nextOffset` says where
  * the next page starts. A page is the hits up to `offset + limit` read again and cut, so deep pages cost more.
  */
-export declare const searchStore: (store: MessageStore, account: AccountKey, request: SearchQuery, messenger?: Saved) => Promise<SearchFound>;
+export declare const searchStore: (store: MessageStore, account: AccountKey, request: SearchQuery, messenger?: Saved, meaning?: MeaningLane) => Promise<SearchFound>;
 export declare const accountOf: (hit: StoredHit) => AccountKey;
 export declare const keyOf: ({ provider, account }: AccountKey) => string;

package/dist/services/messages-archive-search.js

@@ -43,18 +43,18 @@ export const MAX_SEARCH_OFFSET = 1000;
  * Search the store, one page at a time: `offset` skips the hits a page before held, and `nextOffset` says where
  * the next page starts. A page is the hits up to `offset + limit` read again and cut, so deep pages cost more.
  */
-export const searchStore = async (store, account, request, messenger = {}) => {
+export const searchStore = async (store, account, request, messenger = {}, meaning) => {
     const offset = request.offset ?? 0;
     if (!Number.isInteger(offset) || offset < 0 || offset > MAX_SEARCH_OFFSET)
         throw new CliError("validation_error", `offset takes a whole number from 0 to ${MAX_SEARCH_OFFSET}`);
-    const found = await searchPage(store, account, { ...request, limit: request.limit + offset }, messenger);
+    const found = await searchPage(store, account, { ...request, limit: request.limit + offset }, messenger, meaning);
     return {
         ...found,
         items: found.items.slice(offset),
         ...(found.hasMore && offset + request.limit <= MAX_SEARCH_OFFSET ? { nextOffset: offset + request.limit } : {}),
     };
 };
-const searchPage = async (store, account, request, messenger) => {
+const searchPage = async (store, account, request, messenger, meaning) => {
     if (request.signal?.aborted)
         throw new CliError("validation_error", "search was aborted", { reason: "query.aborted", complete: false });
     if (request.thread)
@@ -65,7 +65,7 @@ const searchPage = async (store, account, request, messenger) => {
         throw new CliError("validation_error", "discovery searches the local archive; use backend archive");
     await topUp(store);
     const found = request.discover
-        ? await searchDiscovery(store, account, { ...request, thread: undefined }, messenger)
+        ? await searchDiscovery(store, account, { ...request, thread: undefined }, messenger, meaning)
         : await searchLucene(store, account, request, messenger);
     return withThreads(store, found, request);
 };

package/dist/services/messages-discovery.d.ts

@@ -1,4 +1,27 @@
+import type { Id } from "../domain/models.js";
 import type { MessengerTraits } from "../ports/traits.js";
 import type { AccountKey, MessageStore } from "../store/store.js";
 import type { SearchFound, SearchQuery } from "./messages.js";
-export declare const searchDiscovery: (store: MessageStore, account: AccountKey, asked: SearchQuery, messenger?: Partial<Pick<MessengerTraits, "savedChatId" | "app" | "history" | "provider">>) => Promise<SearchFound>;
+/** How many of the messages nearest in meaning join the fusion: the benchmark's pool (performance/search/evidence-pipeline). */
+export declare const MEANING_DEPTH = 100;
+export type MeaningRanked = (AccountKey & {
+    chatId: Id;
+    id: Id;
+    score: number;
+})[];
+/** Why meaning was not searched, and the step that would make it searchable. */
+export interface MeaningSkipped {
+    reason: string;
+    /** Absent when the model itself failed: nothing the owner runs would fix it. */
+    step?: "models.download" | "conversations.rebuild";
+    model: string;
+}
+/** The messages of these accounts nearest in meaning to the question, best first; or why there are none. */
+export type MeaningLane = (question: string, accounts: AccountKey[]) => Promise<{
+    model: string;
+    ranked: MeaningRanked;
+    partly?: MeaningSkipped;
+} | {
+    skipped: MeaningSkipped;
+}>;
+export declare const searchDiscovery: (store: MessageStore, account: AccountKey, asked: SearchQuery, messenger?: Partial<Pick<MessengerTraits, "savedChatId" | "app" | "history" | "provider">>, meaning?: MeaningLane) => Promise<SearchFound>;

package/dist/services/messages-discovery.js

@@ -5,7 +5,10 @@ import { exhausted, QUERY_LIMITS } from "../search/lucene/types.js";
 import { planQuestion, words } from "../search/question-plan.js";
 import { searchCombined } from "./messages-combined.js";
 import { prepareLucene, searchLucene } from "./messages-search.js";
-export const searchDiscovery = async (store, account, asked, messenger = {}) => {
+/** How many of the messages nearest in meaning join the fusion: the benchmark's pool (performance/search/evidence-pipeline). */
+export const MEANING_DEPTH = 100;
+const RRF_K = 60;
+export const searchDiscovery = async (store, account, asked, messenger = {}, meaning) => {
     // A question's own "?" would parse as a one-character wildcard and send it to strict search (BUG-395).
     const request = asked.ast !== undefined || asked.exact || !asked.text
         ? asked
@@ -25,6 +28,8 @@ export const searchDiscovery = async (store, account, asked, messenger = {}) =>
         return searchLucene(store, account, request, messenger);
     if (terms.length > 64)
         exhausted("discovery terms");
+    // Before the clock starts: loading the model alone takes about a second of the word search's budget.
+    const lane = meaning ? await meaning(text, original.scopeAccounts) : undefined;
     const started = performance.now();
     const check = () => {
         if (request.signal?.aborted)
@@ -115,8 +120,7 @@ export const searchDiscovery = async (store, account, asked, messenger = {}) =>
     const stem = (w) => original.execution.stemmer?.stemToken(w) ?? w;
     const stems = terms.map(stem);
     let bytes = 0;
-    candidates = candidates
-        .map((m) => {
+    const described = (m) => {
         check();
         const parent = parentByLocator.get(m.locator);
         const body = `${parent ? `${parent.text}\n` : ""}${m.text}`;
@@ -137,9 +141,17 @@ export const searchDiscovery = async (store, account, asked, messenger = {}) =>
                 ...(parent ? { parent: parent.locator } : {}),
             },
         };
-    })
+    };
+    candidates = candidates
+        .map(described)
         .sort((a, b) => (b.score ?? 0) - (a.score ?? 0) ||
         Number(parentByLocator.has(b.locator)) - Number(parentByLocator.has(a.locator)));
+    let meaningCandidates = 0;
+    if (lane && "ranked" in lane) {
+        const near = await nearEligible(store, account, request, messenger, lane.ranked, plan.scope);
+        meaningCandidates = near.length;
+        candidates = fusedWithMeaning(candidates, near, described);
+    }
     check();
     const items = await Promise.all(candidates.slice(0, request.limit).map(async ({ fused: _fused, ...m }) => {
         if (!request.context)
@@ -171,7 +183,52 @@ export const searchDiscovery = async (store, account, asked, messenger = {}) =>
                 truncated,
                 terms,
                 queries: [...(plan.changed ? [] : plan.queries), relaxed],
+                ...(lane && "ranked" in lane ? { meaning: { model: lane.model, candidates: meaningCandidates } } : {}),
             },
         },
+        ...(lane && "skipped" in lane ? { meaningSkipped: lane.skipped } : {}),
+        ...(lane && "partly" in lane && lane.partly ? { meaningSkipped: lane.partly } : {}),
     };
 };
+/** The nearest messages the question's filters allow, as hits, nearest first. */
+const nearEligible = async (store, account, request, messenger, ranked, scope) => {
+    const keyOf = (one) => JSON.stringify([one.provider, one.account, one.chatId, one.id]);
+    const only = ranked
+        .slice(0, MEANING_DEPTH)
+        .filter((m) => !request.only || request.only.some((key) => keyOf(key) === keyOf(m)))
+        .map(({ provider, account, chatId, id }) => ({ provider, account, chatId, id }));
+    if (!only.length)
+        return [];
+    const eligible = await searchLucene(store, account, { ...request, text: scope || undefined, ast: undefined, limit: only.length, only, context: 0, thread: undefined }, messenger);
+    const hits = new Map(eligible.items.map((hit) => {
+        const { provider, account } = parseLocator(hit.locator);
+        return [keyOf({ provider, account, chatId: hit.chatId, id: hit.id }), hit];
+    }));
+    return ranked.flatMap((m) => {
+        const hit = hits.get(keyOf(m));
+        return hit ? [{ ...hit, cosine: m.score }] : [];
+    });
+};
+/**
+ * Reciprocal rank fusion of discovery's order and the meaning order, k = 60, as the benchmark's A2 arm: a hit
+ * scores 1 / (k + rank) in each list it is in. Coverage is left out of the fused score, so a message that answers
+ * in other words can rise above one that only repeats them.
+ */
+const fusedWithMeaning = (words, near, described) => {
+    const merged = new Map();
+    const by = (hit, foundBy) => hit.discovery ? { ...hit, discovery: { ...hit.discovery, foundBy } } : hit;
+    words.forEach((hit, rank) => {
+        merged.set(hit.locator, { hit: by(hit, ["words"]), rrf: 1 / (RRF_K + rank + 1) });
+    });
+    near.forEach(({ cosine: _, ...m }, rank) => {
+        const share = 1 / (RRF_K + rank + 1);
+        const held = merged.get(m.locator);
+        if (held) {
+            held.rrf += share;
+            held.hit = by(held.hit, ["words", "meaning"]);
+        }
+        else
+            merged.set(m.locator, { hit: by(described({ ...m, fused: 0 }), ["meaning"]), rrf: share });
+    });
+    return [...merged.values()].sort((a, b) => b.rrf - a.rrf).map(({ hit, rrf }) => ({ ...hit, score: rrf }));
+};

package/dist/services/messages-meaning.d.ts

@@ -0,0 +1,7 @@
+import type { ServiceDeps } from "./deps.js";
+import { type MeaningLane } from "./messages-discovery.js";
+/**
+ * Discovery's meaning half: the default local model over each message's own vector. It runs only when the model is
+ * on disk and vectors exist — never a download, never a remote model — and any failure leaves the words alone.
+ */
+export declare const meaningLane: (deps: ServiceDeps) => MeaningLane;

package/dist/services/messages-meaning.js

@@ -0,0 +1,53 @@
+import { DEFAULT_TEXT_MODEL } from "../embeddings/models.js";
+import { embeddingTarget, withEmbedder } from "./embeddings-maintenance.js";
+import { MEANING_DEPTH } from "./messages-discovery.js";
+import { plainWords } from "./pictures.js";
+/**
+ * Discovery's meaning half: the default local model over each message's own vector. It runs only when the model is
+ * on disk and vectors exist — never a download, never a remote model — and any failure leaves the words alone.
+ */
+export const meaningLane = (deps) => async (question, accounts) => {
+    const target = embeddingTarget(deps, DEFAULT_TEXT_MODEL);
+    const skipped = (reason, step) => ({
+        skipped: { reason, model: target.id, ...(step ? { step } : {}) },
+    });
+    if (!target.installed)
+        return skipped(`${target.id} is not downloaded`, "models.download");
+    const store = await deps.store();
+    const ready = [];
+    const without = [];
+    for (const one of accounts)
+        ((await store.vectors.hasMessageVectors(one, target.key)) ? ready : without).push(one);
+    if (ready.length === 0)
+        return skipped("no message has a vector yet", "conversations.rebuild");
+    const words = plainWords(question);
+    try {
+        const ranked = [];
+        if (words) {
+            await withEmbedder(deps, target, DEFAULT_TEXT_MODEL, {}, async (embedder) => {
+                const [query] = await embedder.embed([words], "query");
+                for (const one of ready) {
+                    const near = await store.vectors.nearestMessages(one, {
+                        model: target.key,
+                        limit: MEANING_DEPTH,
+                        query: query,
+                    });
+                    for (const { chatId, messageId, score } of near)
+                        ranked.push({ ...one, chatId, id: messageId, score });
+                }
+            });
+        }
+        return {
+            model: target.id,
+            ranked: ranked.sort((a, b) => b.score - a.score).slice(0, MEANING_DEPTH),
+            ...(without.length
+                ? {
+                    partly: skipped(`${without.map(({ provider, account }) => `${provider}:${account}`).join(", ")} has no message vectors yet`, "conversations.rebuild").skipped,
+                }
+                : {}),
+        };
+    }
+    catch (error) {
+        return skipped(`${target.id} failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+};

package/dist/services/messages-search.d.ts

@@ -15,6 +15,11 @@ export interface QueryMetadata {
         truncated: boolean;
         terms: string[];
         queries: string[];
+        /** Present when meaning was searched too: the model, and how many of its nearest messages the filters allowed. */
+        meaning?: {
+            model: string;
+            candidates: number;
+        };
     };
     version: 1;
     fieldsVersion: number;

package/dist/services/messages.d.ts

@@ -7,6 +7,7 @@ import { type Operated } from "../sends/guarded.js";
 import type { PermissionKey } from "../sends/permissions.js";
 import type { AccountKey, ChatCompleteness, MatchPlace, StoredHit } from "../store/store.js";
 import { type ServiceDeps } from "./deps.js";
+import type { MeaningSkipped } from "./messages-discovery.js";
 import type { MessageStats, QueryMetadata, SearchCoverage, StatsGrouping } from "./messages-search.js";
 import type { PictureHit } from "./pictures.js";
 import { type SearchAllFound, type SearchAllRequest } from "./search-all.js";
@@ -79,6 +80,8 @@ export type FoundMessage = StoredHit & {
         matchedTerms: string[];
         missingTerms: string[];
         parent?: string;
+        /** Only when meaning was searched too: which of the two lists found it. */
+        foundBy?: ("words" | "meaning")[];
     };
     match?: "words" | "beginnings" | "corrected";
     /** The garbled spellings a `corrected` hit's transcript holds, per query word. */
@@ -116,6 +119,8 @@ export interface SearchFound extends Page<FoundMessage> {
     /** Per chat of the page: how much of its history the store holds. */
     completeness: (ChatCompleteness & AccountKey)[];
     /** `false` while the word index is being built. */
+    /** `--discover` searched by words alone: why meaning was not searched, and the step that would enable it. */
+    meaningSkipped?: MeaningSkipped;
     wordsReady: boolean;
     /** The same for the stems, in strict search: `false` until a stemmed search may run. */
     stemsReady?: boolean;

package/dist/services/messages.js

@@ -12,6 +12,7 @@ import { uploadSize } from "../sends/upload.js";
 import { crossAccount, fromStore, PUSHED } from "./deps.js";
 import { readChatId, storedChatId } from "./message-resolution.js";
 import { accountOf, keyOf, searchStore, statsStore, topUp } from "./messages-archive-search.js";
+import { meaningLane } from "./messages-meaning.js";
 import { searchAll } from "./search-all.js";
 import { searchAllWithMeetings, } from "./search-all-meetings.js";
 import { searchRecordOf } from "./search-record.js";
@@ -31,7 +32,7 @@ export const messagesService = (deps) => {
                 return { ...account, chatId, id };
             })
             : undefined;
-        const found = await searchStore(store, account, only ? { ...query, only } : query, deps.messenger);
+        const found = await searchStore(store, account, only ? { ...query, only } : query, deps.messenger, query.discover && query.kind !== "mail" ? meaningLane(deps) : undefined);
         return { found, server };
     };
     const unifiedMessages = (store, account) => async (query) => {

package/dist/services/search-catchup.js

@@ -36,6 +36,7 @@ export const catchUpSearch = async (deps, chat, options, signal, clock = Date.no
             maxMessages,
             check,
             threads: 1,
+            messages: false,
         });
         check();
         return {

package/dist/store/sqlite/message-chunks.d.ts

@@ -0,0 +1,44 @@
+import type { StoreContext } from "./open.js";
+/** A live message with text, and the chunk and vector it already has, if any. */
+export interface MessageToEmbed {
+    messagePk: number;
+    text: string;
+    sentAt: number;
+    hash: string | null;
+    embedded: boolean;
+}
+/** Live messages with text of the account, by pk after `after`; the caller compares hashes. */
+export declare const messagesToEmbed: ({ orm }: StoreContext, accountPk: number, { chatKey, model, after, limit }: {
+    chatKey?: number;
+    model: string;
+    after: number;
+    limit: number;
+}) => MessageToEmbed[];
+/** Makes each hash its message's chunk; the vectors are saved apart, by `saveVectors`. */
+export declare const saveMessageChunks: (context: StoreContext, accountPk: number, chunks: {
+    messagePk: number;
+    hash: string;
+    length: number;
+    occurredAt: number;
+}[]) => void;
+/**
+ * Drops the chunks of the account's messages that are gone, deleted or emptied, with the vectors nothing else
+ * uses: no trigger does it for messages, and a deleted message keeps no trace of its text.
+ */
+export declare const dropGoneMessageChunks: (context: StoreContext, accountPk: number) => number;
+/** Whether any message of the account has a vector of `model`. */
+export declare const hasMessageVectors: ({ orm }: StoreContext, accountPk: number, model: string) => boolean;
+export interface NearestMessage {
+    chatId: string;
+    messageId: string;
+    score: number;
+}
+/** The account's live messages nearest to `query` by their vector of `model`, best first: a scan of every vector. */
+export declare const nearestMessages: ({ orm }: StoreContext, accountPk: number, { chatKey, model, since, before, limit, query, }: {
+    chatKey?: number;
+    model: string;
+    since?: number;
+    before?: number;
+    limit: number;
+    query: Float32Array;
+}) => NearestMessage[];