@wirecat/cli-core 0.270.0 → 0.272.0

package/dist/services/embeddings.js

@@ -1,631 +1,8 @@
-import { CliError } from "@wirecat/cli-base";
-import { CHUNK_CHARS, chunkHash, chunkTextOf } from "../conversations/chunks.js";
-import { RULES_VERSION } from "../conversations/link.js";
-import { formatLocator, parseLocator } from "../domain/locator.js";
-import { defaultThreads, isTextModelInstalled, textModelsDirectory } from "../embeddings/embed.js";
-import { DEFAULT_TEXT_MODEL, meaningFloor, textModel } from "../embeddings/models.js";
-import { remoteKey } from "../embeddings/remote.js";
-import { CONTEXT_MESSAGES, CONTEXT_MINUTES } from "../images/context/index.js";
-import { registeredImageProviders } from "../images/registry.js";
-import { QUERY_LIMITS, searchError } from "../search/lucene/types.js";
-import { threadConversation, } from "../store/store.js";
-import { conversationsService } from "./conversations.js";
-import { storedChatId } from "./messages.js";
-import { prepareLucene } from "./messages-search.js";
-import { MAIL } from "./search-kind.js";
-import { refreshSearch } from "./search-refresh.js";
-/** How much one run without `--chat` does at most, by default. */
-export const REFRESH_BOUNDS = { maxChats: 20, maxChunks: 2_000 };
-/** A vector's model: the provider, the model and its size — vectors of two of them never mix. */
-export const vectorModelKey = (model) => `local:${model.id}:${model.dims}`;
-/** Chunks per batch for one local session; one short transaction writes each batch. */
-const PER_SESSION = 8;
-/** Texts per remote request; `concurrency` of them run at once. */
-const PER_REQUEST = 256;
-/** How deep each list is read before the two are merged. */
-const CANDIDATES = 50;
-/** Messages the word search reads: one long thread can hold many of them. */
-const WORD_HITS = 200;
-/** Reciprocal rank fusion's usual constant (phase 5 E9). */
-const RRF_K = 60;
-/** e5's tokenizer reads ~3.7 characters a token in Russian; 3 keeps the bound above the real count. */
-const CHARS_PER_TOKEN_AT_LEAST = 3;
-export const embeddingsService = (deps) => {
-    const command = deps.messenger.app.command;
-    const notDownloaded = (id) => new CliError("not_found", `${id} is not downloaded — \`${command} models text download ${id}\``, {
-        reason: "model.not-downloaded",
-        model: id,
-    });
-    const notBuiltError = (chatId) => new CliError("not_found", `chat ${chatId} has no conversations yet — \`${command} conversations build --chat ${chatId}\``);
-    const resolve = (choice = DEFAULT_TEXT_MODEL) => {
-        if (typeof choice !== "string") {
-            const { remote, apiKey, concurrency = 4 } = choice;
-            return {
-                id: `${remote.provider}:${remote.model}`,
-                key: remoteKey(remote),
-                dims: remote.dims,
-                speed: null,
-                price: remote.price,
-                installed: true,
-                perBatch: (_workers) => PER_REQUEST * concurrency,
-                open: async () => (await import("../embeddings/remote.js")).openRemote(remote, apiKey, { concurrency }),
-            };
-        }
-        const model = textModel(choice);
-        const directory = textModelsDirectory(deps.env);
-        return {
-            id: model.id,
-            key: vectorModelKey(model),
-            dims: model.dims,
-            speed: model.chunksPerSecond,
-            price: undefined,
-            installed: isTextModelInstalled(model, directory),
-            perBatch: (workers) => PER_SESSION * Math.max(1, workers),
-            open: async ({ workers = 1, threads, apart = false, } = {}) => {
-                if (!isTextModelInstalled(model, directory))
-                    throw notDownloaded(model.id);
-                if (apart) {
-                    const { openProcess } = await import("../embeddings/process.js");
-                    return openProcess(model, directory, { threads: threads ?? defaultThreads() });
-                }
-                const { openPool } = await import("../embeddings/pool.js");
-                return openPool(model, directory, { workers, ...(threads ? { threads } : {}) });
-            },
-        };
-    };
-    /**
-     * The conversations holding messages that share a word with the query, best message first. Every word
-     * is one side of an OR, so a sentence ranks what shares any of it, and nothing in it is read as a filter.
-     * Only whole words and their beginnings count: a corrected spelling or a piece of a word would pass a
-     * near-miss off as a match, which is what the meaning half is for.
-     */
-    const byWords = async (query, prepared, conversations, since) => {
-        const words = [...new Set(query.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [])];
-        const outside = new Map();
-        if (!words.length || !prepared.scopeAccounts.length)
-            return { hits: [], outside, sources: new Map() };
-        const store = await deps.store();
-        const lexical = await prepareLucene(store, await deps.account(), {
-            text: words.map((word) => (word.length >= 3 ? `${word}*` : `"${word}"`)).join(" OR "),
-            accounts: prepared.scopeAccounts,
-            limit: WORD_HITS,
-        }, deps.messenger);
-        if (!store.matchQuery)
-            throw new CliError("validation_error", "the store does not support strict search");
-        const { items } = await store.matchQuery({
-            ...lexical.execution,
-            ...(prepared.selectedChat ? { chat: prepared.selectedChat } : {}),
-            ...(conversations === undefined ? {} : { conversationIds: conversations }),
-            ...(since === undefined ? {} : { conversationSince: Date.parse(since) }),
-        });
-        const summaries = new Map();
-        const sources = new Map();
-        for (const account of prepared.scopeAccounts) {
-            const refs = items.flatMap((hit, index) => {
-                const at = parseLocator(hit.locator);
-                return at.provider === account.provider && at.account === account.account ? [{ hit, index }] : [];
-            });
-            const found = await store.conversationsOfMessages(account, refs.map(({ hit }) => ({ chatId: hit.chatId, messageId: hit.id })));
-            const chats = new Set();
-            refs.forEach(({ hit, index }, at) => {
-                const summary = found[at];
-                if (summary) {
-                    summaries.set(index, summary);
-                    sources.set(summary.id, account);
-                }
-                else
-                    chats.add(hit.chatId);
-            });
-            outside.set(JSON.stringify(account), { account, chats });
-        }
-        const ranked = items.flatMap(({ id, score }, index) => {
-            const summary = summaries.get(index);
-            return summary ? [{ summary, chunk: { firstMessageId: id, lastMessageId: id }, score }] : [];
-        });
-        // Mail in its own tables has no conversations: each email answers as its thread, ranked with the messages.
-        const mailAccounts = prepared.scopeAccounts.filter(({ provider }) => provider === MAIL);
-        if (mailAccounts.length && conversations === undefined) {
-            const mail = await store
-                .matchQuery({
-                ...lexical.execution,
-                accounts: mailAccounts,
-                corpus: "mail",
-                ...(prepared.selectedChat ? { chat: prepared.selectedChat } : {}),
-            })
-                .catch((error) => {
-                if (error instanceof CliError && error.code === "validation_error")
-                    return { items: [] };
-                throw error;
-            });
-            for (const account of mailAccounts) {
-                const found = mail.items.filter((hit) => {
-                    const at = parseLocator(hit.locator);
-                    return at.account === account.account && (since === undefined || hit.timestamp >= since);
-                });
-                if (!found.length)
-                    continue;
-                const { id } = await store.storedAccount(account);
-                const places = new Map((await store.mail.inThreads(id, found.map((hit) => hit.id))).map((place) => [place.externalId, place]));
-                for (const hit of found) {
-                    const place = places.get(hit.id);
-                    if (!place)
-                        continue;
-                    const summary = threadConversation(place, Date.now());
-                    sources.set(summary.id, account);
-                    ranked.push({ summary, chunk: { firstMessageId: hit.id, lastMessageId: hit.id }, score: hit.score });
-                }
-            }
-            ranked.sort((a, b) => (b.score ?? Number.NEGATIVE_INFINITY) - (a.score ?? Number.NEGATIVE_INFINITY));
-        }
-        const seen = new Set();
-        const hits = ranked.flatMap(({ summary, chunk }) => {
-            if (seen.has(summary.id))
-                return [];
-            seen.add(summary.id);
-            return [{ summary, chunk }];
-        });
-        return { hits, outside, sources };
-    };
-    const readinessOf = async (chatId, key) => {
-        const store = await deps.store();
-        const account = await deps.account();
-        return (await store.readiness(account, { ...(chatId === undefined ? {} : { chatId }), model: key })).map(shaped);
-    };
-    const found = async (chat) => {
-        const store = await deps.store();
-        const account = await deps.account();
-        const chatId = await storedChatId(deps.messenger, chat, store, account);
-        if (!(await store.conversationState(account, chatId))?.builtAt)
-            throw notBuiltError(chatId);
-        return { store, account, chatId };
-    };
-    /** The server's warm model when there is one, kept open after; otherwise opened here and closed. */
-    const withEmbedder = async (target, choice, { workers = 1, threads }, work) => {
-        const warm = typeof choice !== "object" ? deps.embedders : undefined;
-        const embedder = warm
-            ? await warm.get(target.key, () => target.open({ apart: true }))
-            : await target.open({ workers, ...(threads ? { threads } : {}) });
-        try {
-            return await work(embedder);
-        }
-        finally {
-            if (!warm)
-                await embedder.close();
-        }
-    };
-    const embedInto = async (chatId, target, embedder, { workers = 1, maxChunks = Number.POSITIVE_INFINITY, progress, check, }) => {
-        const store = await deps.store();
-        const account = await deps.account();
-        let embedded = 0;
-        let skipped = 0;
-        let left = (await store.vectorStatus(account, chatId, target.key)).chunks;
-        let after;
-        while (embedded + skipped < maxChunks) {
-            check?.();
-            const batch = await store.chunksToEmbed(account, chatId, target.key, {
-                limit: Math.min(target.perBatch(workers), maxChunks - embedded - skipped),
-                ...(after ? { after } : {}),
-            });
-            if (batch.length === 0)
-                break;
-            after = batch.at(-1)?.hash;
-            const ready = batch.flatMap(({ hash, lines }) => {
-                const text = chunkTextOf(lines);
-                return chunkHash(text) === hash ? [{ hash, text }] : [];
-            });
-            skipped += batch.length - ready.length;
-            const vectors = await embedder.embed(ready.map(({ text }) => text), "passage");
-            check?.();
-            await store.saveVectors(target.key, target.dims, ready.map(({ hash }, index) => ({ hash, vector: vectors[index] })));
-            embedded += ready.length;
-            left = Math.max(0, left - batch.length);
-            progress?.(embedded, left);
-        }
-        embedded += await embedImages(chatId, target, embedder, check);
-        return { chat: chatId, model: target.id, embedded, skipped };
-    };
-    /**
-     * One chunk per image of the chat: the text its first image provider gives it, embedded like a message. The
-     * chunk's hash names the provider and its version, so a newer provider describes every image again.
-     */
-    const embedImages = async (chatId, target, embedder, check) => {
-        const providers = registeredImageProviders();
-        if (providers.length === 0)
-            return 0;
-        const store = await deps.store();
-        const account = await deps.account();
-        const images = await store.imagesToDescribe(account, chatId, {
-            messages: CONTEXT_MESSAGES,
-            withinMs: CONTEXT_MINUTES * 60_000,
-        });
-        const described = [];
-        for (const image of images) {
-            check?.();
-            for (const provider of providers) {
-                const { text } = await provider.describe({ kind: image.kind, name: image.name, mime: image.mime, localPath: image.localPath }, {
-                    chatTitle: image.chatTitle,
-                    sender: image.sender,
-                    caption: image.caption,
-                    sentAt: image.sentAt,
-                    before: image.before,
-                    after: image.after,
-                });
-                if (text?.trim()) {
-                    described.push({
-                        attachmentPk: image.attachmentPk,
-                        hash: chunkHash(`${provider.id}:${provider.version}\n${text}`),
-                        text,
-                        occurredAt: image.sentAt,
-                    });
-                    break;
-                }
-            }
-        }
-        const have = await store.embeddedHashes(target.key, described.map(({ hash }) => hash));
-        const missing = [...new Map(described.filter(({ hash }) => !have.has(hash)).map((one) => [one.hash, one])).values()];
-        for (let at = 0; at < missing.length; at += PER_REQUEST) {
-            const batch = missing.slice(at, at + PER_REQUEST);
-            const vectors = await embedder.embed(batch.map(({ text }) => text), "passage");
-            check?.();
-            await store.saveVectors(target.key, target.dims, batch.map(({ hash }, index) => ({ hash, vector: vectors[index] })));
-        }
-        await store.saveImageChunks(account, chatId, described.map(({ attachmentPk, hash, text, occurredAt }) => ({
-            attachmentPk,
-            hash,
-            length: text.length,
-            occurredAt,
-        })), described.map(({ attachmentPk }) => attachmentPk));
-        return missing.length;
-    };
-    return {
-        status: async (chat, choice) => {
-            const { store, account, chatId } = await found(chat);
-            const target = resolve(choice);
-            const { chunks, embedded } = await store.vectorStatus(account, chatId, target.key);
-            const left = chunks - embedded;
-            const tokensAtMost = Math.ceil((left * CHUNK_CHARS) / CHARS_PER_TOKEN_AT_LEAST);
-            return {
-                chat: chatId,
-                model: target.id,
-                chunks,
-                embedded,
-                left,
-                estimateSeconds: target.speed === null ? null : Math.ceil(left / target.speed),
-                tokensAtMost,
-                ...(target.price
-                    ? { priceAtMost: { usd: (tokensAtMost / 1e6) * target.price.perMillion, read: target.price.read } }
-                    : {}),
-            };
-        },
-        embed: async (chat, { model: choice, workers = 1, threads, maxChunks, progress }) => {
-            const { chatId } = await found(chat);
-            const target = resolve(choice);
-            return withEmbedder(target, choice, { workers, threads }, (embedder) => embedInto(chatId, target, embedder, { workers, maxChunks, progress }));
-        },
-        search: async (query, { chat, model: choice, since, limit, filter, source, timezone, syncFirst, signal, refresh }) => {
-            if (!query.trim())
-                searchError("query.invalid-syntax", { start: 0, end: query.length }, "give words to search for");
-            if (new TextEncoder().encode(query).length > QUERY_LIMITS.bytes)
-                searchError("query.too-large", { start: 0, end: query.length }, "query is longer than 8 KiB");
-            const refreshed = await refreshSearch(deps, {
-                chat,
-                limit,
-                syncFirst,
-                signal,
-                language: "lucene",
-                text: filter,
-                source,
-                timezone,
-            });
-            const locallyRefreshed = refresh ? await embeddingsService(deps).refresh(refresh) : undefined;
-            const store = await deps.store();
-            const account = await deps.account();
-            const target = resolve(choice);
-            const stop = Date.now() + 200;
-            await store.fillSearchIndex({ until: () => Date.now() >= stop });
-            await store.fillStems({ until: () => Date.now() >= stop });
-            const prepared = await prepareLucene(store, account, {
-                ...(filter === undefined ? {} : { text: filter }),
-                ...(source === undefined ? {} : { source }),
-                ...(chat === undefined ? {} : { chat }),
-                ...(timezone === undefined ? {} : { timezone }),
-                limit,
-            }, deps.messenger);
-            if (filter !== undefined && !store.conversationEligibility)
-                throw new CliError("validation_error", "the store does not support conversation filters — upgrade cli-core");
-            const eligibility = filter === undefined ? undefined : await store.conversationEligibility?.(prepared.execution);
-            const selected = prepared.selectedChat;
-            const scopeOf = (one) => selected
-                ? selected.account.provider === one.provider && selected.account.account === one.account
-                    ? { chatId: selected.chatId }
-                    : null
-                : {};
-            const words = await byWords(query, prepared, eligibility?.conversations, since);
-            const meaning = [];
-            const sources = words.sources;
-            if (target.installed && prepared.scopeAccounts.length && eligibility?.conversations.length !== 0) {
-                await withEmbedder(target, choice, {}, async (embedder) => {
-                    const [vector] = await embedder.embed([query], "query");
-                    for (const one of prepared.scopeAccounts) {
-                        const scope = scopeOf(one);
-                        if (scope === null)
-                            continue;
-                        const hits = await store.nearestConversations(one, {
-                            ...scope,
-                            ...(since === undefined ? {} : { since }),
-                            model: target.key,
-                            limit: Math.max(limit, CANDIDATES),
-                            query: vector,
-                            ...(eligibility ? { conversations: eligibility.conversations } : {}),
-                        });
-                        for (const hit of hits)
-                            sources.set(hit.summary.id, one);
-                        meaning.push(...hits);
-                        const images = await store.nearestImages(one, {
-                            ...scope,
-                            ...(since === undefined ? {} : { since }),
-                            model: target.key,
-                            limit: Math.max(limit, CANDIDATES),
-                            query: vector,
-                        });
-                        for (const hit of images) {
-                            if (eligibility && !eligibility.conversations.includes(hit.summary.id))
-                                continue;
-                            sources.set(hit.summary.id, one);
-                            meaning.push(hit);
-                        }
-                    }
-                });
-            }
-            meaning.sort((a, b) => b.score - a.score);
-            const scoped = [];
-            const elsewhere = [];
-            for (const one of prepared.scopeAccounts) {
-                const scope = scopeOf(one);
-                if (scope === null)
-                    continue;
-                const eligibleChats = eligibility?.chats
-                    .filter(({ account }) => account.provider === one.provider && account.account === one.account)
-                    .map(({ chat }) => chat);
-                const rows = eligibleChats === undefined
-                    ? await store.readiness(one, { ...scope, model: target.key })
-                    : (await Promise.all(eligibleChats.map((chatId) => store.readiness(one, { chatId, model: target.key })))).flat();
-                const readiness = searchReadiness(rows.map(shaped), target.installed, words.outside.get(JSON.stringify(one))?.chats ?? new Set());
-                scoped.push({ source: one, readiness });
-                const other = await store.embeddedOnlyElsewhere(one, { ...scope, model: target.key });
-                elsewhere.push(...other
-                    .filter((id) => eligibleChats === undefined || eligibleChats.includes(id))
-                    .map((id) => prepared.scopeAccounts.length > 1
-                    ? [one.provider, one.account, id].map(encodeURIComponent).join("/")
-                    : id));
-            }
-            const readiness = { searchedByMeaning: [], wordsOnly: [], partial: [], stale: [], notBuilt: [] };
-            for (const key of ["searchedByMeaning", "wordsOnly", "partial", "stale", "notBuilt"])
-                readiness[key] = [
-                    ...new Set(scoped.flatMap(({ source, readiness }) => readiness[key].map((id) => scoped.length > 1 ? [source.provider, source.account, id].map(encodeURIComponent).join("/") : id))),
-                ].sort();
-            if (scoped.length > 1)
-                readiness.scopes = scoped;
-            return {
-                accounts: prepared.scopeAccounts,
-                model: target.id,
-                meaning: target.installed ? "searched" : "unavailable",
-                hits: fused(meaning, words.hits, meaningFloor(target.key))
-                    .slice(0, limit)
-                    .map((hit) => {
-                    const source = sources.get(hit.summary.id);
-                    return {
-                        ...hit,
-                        source,
-                        locator: formatLocator({ ...source, chat: hit.summary.chatId, message: hit.chunk.firstMessageId }),
-                    };
-                }),
-                readiness,
-                ...(locallyRefreshed ? { prepared: locallyRefreshed } : {}),
-                ...(refreshed
-                    ? { refreshed, coverage: { state: refreshed.complete ? "partial" : "stale" } }
-                    : {}),
-                embeddedOnlyElsewhere: elsewhere,
-            };
-        },
-        related: async (chat, message, { model: choice, limit }) => {
-            const store = await deps.store();
-            const account = await deps.account();
-            const target = resolve(choice);
-            const chatId = await storedChatId(deps.messenger, chat, store, account);
-            const [source] = await store.conversationsOfMessages(account, [{ chatId, messageId: message }]);
-            if (!source) {
-                if (!(await store.conversationState(account, chatId))?.builtAt)
-                    throw notBuiltError(chatId);
-                throw new CliError("not_found", `message ${message} is in no conversation of chat ${chatId}`);
-            }
-            const { vectors } = await store.conversationVectors(account, source.id, target.key);
-            if (vectors.length === 0) {
-                const [own] = await readinessOf(chatId, target.key);
-                const step = own && isStale(own) ? "build" : "embed";
-                throw new CliError("not_found", `conversation ${source.id} has no vector of ${target.id} that matches its messages now — ` +
-                    `\`${command} conversations ${step} --chat ${chatId}\``);
-            }
-            const hits = await store.nearestConversations(account, {
-                model: target.key,
-                limit: Math.max(limit, CANDIDATES),
-                query: meanOf(vectors),
-                exclude: source.id,
-            });
-            return {
-                model: target.id,
-                source,
-                hits: hits
-                    .slice(0, limit)
-                    .map(({ summary, chunk, score, stale }) => ({ summary, chunk, score, by: ["meaning"], stale })),
-                readiness: searchReadiness(await readinessOf(undefined, target.key), true, new Set()),
-            };
-        },
-        readiness: async ({ chat, model: choice }) => {
-            const store = await deps.store();
-            const account = await deps.account();
-            const target = resolve(choice);
-            const chatId = chat === undefined ? undefined : await storedChatId(deps.messenger, chat, store, account);
-            const chats = await readinessOf(chatId, target.key);
-            if (chatId !== undefined)
-                return { model: target.id, chats };
-            return { model: target.id, chats, unbuiltGroups: (await store.unbuiltGroups(account)).length };
-        },
-        refresh: async ({ chat, model: choice = DEFAULT_TEXT_MODEL, build = true, embed = true, maxChats = REFRESH_BOUNDS.maxChats, maxChunks = REFRESH_BOUNDS.maxChunks, workers, threads, progress, maxMessages, check, }) => {
-            check?.();
-            const store = await deps.store();
-            const account = await deps.account();
-            const target = resolve(choice);
-            if (embed && !build && !target.installed)
-                throw notDownloaded(target.id);
-            const chatId = chat === undefined ? undefined : await storedChatId(deps.messenger, chat, store, account);
-            const scan = async () => ({
-                chats: (await readinessOf(chatId, target.key)).sort(byOldestBuild),
-                unbuilt: chatId === undefined ? await store.unbuiltGroups(account) : [],
-            });
-            const built = [];
-            if (build) {
-                const { chats, unbuilt } = await scan();
-                const queue = [...chats.filter(needsBuild).map(({ chat }) => chat), ...unbuilt].slice(0, maxChats);
-                const conversations = conversationsService(deps);
-                for (const id of queue) {
-                    try {
-                        check?.();
-                        const one = await conversations.build(id, { maxMessages, check });
-                        built.push(one);
-                        progress?.(`chat ${id}: ${one.messages} messages → ${one.conversations} conversations`);
-                    }
-                    catch (error) {
-                        // Every message of a once-built chat deleted: nothing to build, and it stays in `left`.
-                        if (!(error instanceof CliError && error.code === "not_found"))
-                            throw error;
-                    }
-                }
-            }
-            const embedded = [];
-            if (embed && target.installed) {
-                const fresh = new Set(built.map(({ chat }) => chat));
-                const queue = (await scan()).chats
-                    .filter(needsEmbed)
-                    .sort((a, b) => Number(fresh.has(b.chat)) - Number(fresh.has(a.chat)))
-                    .slice(0, maxChats);
-                if (queue.length > 0) {
-                    await withEmbedder(target, choice, { workers, threads }, async (embedder) => {
-                        let budget = maxChunks;
-                        for (const { chat } of queue) {
-                            if (budget <= 0)
-                                break;
-                            check?.();
-                            const done = await embedInto(chat, target, embedder, { workers, maxChunks: budget, check });
-                            budget -= done.embedded + done.skipped;
-                            embedded.push(done);
-                            progress?.(`chat ${chat}: ${done.embedded} chunks embedded with ${target.id}`);
-                        }
-                    });
-                }
-            }
-            const after = await scan();
-            const rebuild = build ? [...after.chats.filter(needsBuild).map(({ chat }) => chat), ...after.unbuilt] : [];
-            const pending = new Set(rebuild);
-            return {
-                model: target.id,
-                modelAvailable: target.installed,
-                built,
-                embedded,
-                left: [
-                    ...rebuild.map((chat) => ({ chat, needs: "build" })),
-                    ...(embed
-                        ? after.chats
-                            .filter((one) => needsEmbed(one) && !pending.has(one.chat))
-                            .map(({ chat }) => ({ chat, needs: "embed" }))
-                        : []),
-                ],
-            };
-        },
-        clear: async (chat, choice) => {
-            const store = await deps.store();
-            const account = await deps.account();
-            const chatId = await storedChatId(deps.messenger, chat, store, account);
-            const cleared = await store.clearVectors(account, chatId, choice === undefined ? undefined : resolve(choice).key);
-            return { chat: chatId, cleared };
-        },
-    };
-};
-/** Reciprocal rank fusion of the two lists: each conversation scores 1 / (k + rank) in every list it is in. */
-const fused = (meaning, words, minimumCosine) => {
-    const merged = new Map();
-    meaning
-        .filter(({ score }) => score > minimumCosine)
-        .forEach(({ summary, chunk, score, stale, matchedIn }, index) => {
-        merged.set(matchedIn ? `${summary.id}#${chunk.firstMessageId}` : summary.id, {
-            summary,
-            chunk,
-            score,
-            by: ["meaning"],
-            stale,
-            ...(matchedIn ? { matchedIn } : {}),
-            fused: 1 / (RRF_K + index + 1),
-        });
-    });
-    words.forEach(({ summary, chunk }, index) => {
-        const share = 1 / (RRF_K + index + 1);
-        const held = merged.get(summary.id);
-        if (held) {
-            held.fused += share;
-            held.by.push("words");
-        }
-        else
-            merged.set(summary.id, { summary, chunk, score: null, by: ["words"], stale: false, fused: share });
-    });
-    return [...merged.values()]
-        .sort((a, b) => b.fused - a.fused || (b.score ?? -2) - (a.score ?? -2))
-        .map(({ fused: _, ...hit }) => hit);
-};
-const needsBuild = (one) => one.graph === null || isStale(one);
-const needsEmbed = (one) => one.graph !== null && one.vectors.missing > 0;
-/** Never finished first, then the oldest build: the longest-waiting change goes first. */
-const byOldestBuild = (a, b) => (a.graph?.builtAt ?? "").localeCompare(b.graph?.builtAt ?? "");
-/** The chats in readiness by how a meaning search with these vectors could see them, and the word matches outside any. */
-const searchReadiness = (chats, meaning, outside) => {
-    const built = new Set(chats.filter(({ state }) => state !== "not-built").map(({ chat }) => chat));
-    const ids = (keep) => chats.filter(keep).map(({ chat }) => chat);
-    const byMeaning = (one) => meaning && built.has(one.chat) && one.state !== "words-only" && one.vectors.chunks > 0;
-    return {
-        searchedByMeaning: ids(byMeaning),
-        wordsOnly: ids((one) => built.has(one.chat) && !byMeaning(one)),
-        partial: ids((one) => byMeaning(one) && one.vectors.missing > 0),
-        stale: ids((one) => built.has(one.chat) && isStale(one)),
-        notBuilt: [
-            ...new Set([...ids((one) => one.state === "not-built"), ...[...outside].filter((id) => !built.has(id))]),
-        ].sort(),
-    };
-};
-/** Unit vectors' mean, made unit length again so the dot product stays a cosine. */
-const meanOf = (vectors) => {
-    const mean = new Float32Array(vectors[0]?.length ?? 0);
-    for (const vector of vectors) {
-        for (let index = 0; index < mean.length; index++)
-            mean[index] = mean[index] + vector[index];
-    }
-    const length = Math.hypot(...mean) || 1;
-    return mean.map((value) => value / length);
-};
-const isStale = ({ graph, pending, vectors }) => Boolean(graph?.outdatedRules) || pending.new + pending.edited + pending.deleted > 0 || vectors.stale > 0;
-const shaped = ({ chatId, builtAt, algorithmVersion, pending, vectors }) => {
-    const { embedded, ...counts } = vectors;
-    const graph = builtAt === null
-        ? null
-        : { builtAt, rulesVersion: algorithmVersion, outdatedRules: algorithmVersion !== RULES_VERSION };
-    const one = { chat: chatId, state: "ready", graph, pending, vectors: counts };
-    const state = graph === null
-        ? "not-built"
-        : counts.chunks > 0 && embedded === 0
-            ? "words-only"
-            : isStale(one)
-                ? "stale"
-                : counts.missing > 0
-                    ? "partial"
-                    : "ready";
-    return { ...one, state };
-};
+import { embeddingsMaintenance } from "./embeddings-maintenance.js";
+import { embeddingsSearch } from "./embeddings-search.js";
+export { REFRESH_BOUNDS } from "./embeddings-maintenance.js";
+export const embeddingsService = (deps) => ({
+    ...embeddingsMaintenance(deps),
+    ...embeddingsSearch(deps),
+});
 //# sourceMappingURL=embeddings.js.map