@wirecat/cli-core 0.433.0 → 0.435.0

package/dist/services/search-all.js

@@ -1,18 +1,38 @@
 import { CliError } from "@wirecat/cli-base";
+import { parseLucene } from "@wirecat/cli-base/query";
 import { parseLocator } from "../domain/locator.js";
 import { INTERNAL_NOTES, kindIn } from "../domain/providers.js";
+import { runEmbedders } from "../embeddings/embed.js";
+import { DEFAULT_TEXT_MODEL } from "../embeddings/models.js";
+import { MESSAGE_PARSING } from "../search/lucene/types.js";
+import { planQuestion } from "../search/question-plan.js";
 import { excerpt } from "./excerpt.js";
 import { MAX_SEARCH_OFFSET, searchStore } from "./messages-archive-search.js";
+import { nearEligible } from "./messages-discovery.js";
 import { notesScopeOf, RRF_K, searchNotes } from "./notes-search.js";
 export const RESOURCES_SEARCHED = ["messages", "mail", "notes"];
 /** A query a resource cannot answer is skipped with its reason, never a failure of the whole search. */
 const skippable = (error) => error instanceof CliError && (error.code === "validation_error" || error.code === "not_found");
+/**
+ * The filters the meaning hits must pass, or `undefined` when they cannot be followed: a NOT anywhere (meaning would
+ * bring the excluded back), or a filter under OR or in a group, which the plain list of filters would turn into AND.
+ */
+const filtersBeside = (text) => {
+    const { root } = parseLucene(text, MESSAGE_PARSING);
+    const followable = (node, required) => node.kind === "boolean"
+        ? node.clauses.every(({ node: inner, occur }) => occur !== "must_not" && followable(inner, false))
+        : node.kind === "predicate" && (required || node.field === "text" || node.field === "exact");
+    const top = root.kind === "boolean" ? root.clauses : [{ node: root, occur: "must" }];
+    return top.every(({ node, occur }) => occur !== "must_not" && followable(node, occur === "must"))
+        ? planQuestion(text).scope
+        : undefined;
+};
 /**
  * `search all`: messages, mail and notes from the local store — messages also from the messenger's
  * server, as `search messages` asks it — each list best first, merged by reciprocal rank so no
  * resource's own scores have to be compared with another's.
  */
-export const searchAll = async (store, account, request, messenger = {}, searchMessages = (query) => searchStore(store, account, query, messenger)) => {
+export const searchAll = async (store, account, request, messenger = {}, searchMessages = (query) => searchStore(store, account, query, messenger), meaning) => {
     if (!request.text.trim())
         throw new CliError("validation_error", "say what to find");
     const offset = request.offset ?? 0;
@@ -28,6 +48,7 @@ export const searchAll = async (store, account, request, messenger = {}, searchM
     const skipped = [];
     let hasMore = false;
     let notes;
+    let messagesBy;
     let server;
     const messagesOf = async (kind) => {
         const given = request.scope?.[kind];
@@ -60,7 +81,7 @@ export const searchAll = async (store, account, request, messenger = {}, searchM
             if (!skipped.some(({ source }) => source === one.source))
                 skipped.push(one);
         accounts.push(...read.map(({ provider, account }) => ({ resource: kind, provider, account })));
-        return found.items.map((hit) => {
+        const itemOf = (hit, foundBy) => {
             const at = parseLocator(hit.locator);
             return {
                 kind: kind === "mail" ? "mail" : "message",
@@ -74,55 +95,97 @@ export const searchAll = async (store, account, request, messenger = {}, searchM
                     ? { name: hit.email.from.name ?? hit.email.from.address, ref: hit.email.from.ref }
                     : { name: hit.senderName, ref: null },
                 ...(hit.email ? { email: hit.email } : {}),
-                foundBy: ["words"],
+                foundBy,
             };
+        };
+        const byWords = found.items.map((hit) => itemOf(hit, ["words"]));
+        // Server hits are only those the server returned; the local vectors would add others.
+        if (kind === "mail" || !meaning || request.exact || request.backend === "server")
+            return byWords;
+        const filters = filtersBeside(request.text);
+        const lane = filters === undefined
+            ? {
+                skipped: {
+                    reason: "the query uses NOT, or a filter under OR or in a group, which meaning cannot follow",
+                    model: DEFAULT_TEXT_MODEL,
+                },
+            }
+            : await meaning(request.text, read, embedders);
+        if ("skipped" in lane) {
+            messagesBy = { by: "words", meaningSkipped: lane.skipped };
+            return byWords;
+        }
+        messagesBy = { by: "words and meaning", ...(lane.partly ? { meaningSkipped: lane.partly } : {}) };
+        const near = await nearEligible(store, account, query, messenger, lane.ranked, filters ?? "");
+        // One list per resource, as notes give theirs, so messages get no second vote against mail.
+        const fused = new Map();
+        byWords.forEach((item, rank) => {
+            fused.set(item.ref, { item, score: 1 / (RRF_K + rank + 1) });
         });
+        near.forEach((hit, rank) => {
+            const held = fused.get(hit.locator);
+            if (held)
+                held.item.foundBy = ["words", "meaning"];
+            fused.set(hit.locator, {
+                item: held?.item ?? itemOf(hit, ["meaning"]),
+                score: (held?.score ?? 0) + 1 / (RRF_K + rank + 1),
+            });
+        });
+        return [...fused.values()].sort((a, b) => b.score - a.score).map(({ item }) => item);
     };
-    for (const resource of only) {
-        try {
-            if (resource === "notes") {
-                const chosen = request.scope?.notes;
-                const found = await searchNotes(store, request.text, {
-                    limit: window,
-                    command,
-                    ...(chosen === undefined ? {} : notesScopeOf(chosen)),
-                    ...(request.exact ? { exact: true } : {}),
-                    ...(request.timezone === undefined ? {} : { timezone: request.timezone }),
-                    ...(request.env === undefined ? {} : { env: request.env }),
-                });
-                hasMore ||= found.has_more;
-                notes = {
-                    by: found.by,
-                    ...(found.meaning_skipped === undefined ? {} : { meaningSkipped: found.meaning_skipped }),
-                };
-                lists.push(found.hits.map((hit) => ({
-                    kind: hit.kind ?? "note",
-                    ref: hit.ref,
-                    provider: hit.provider ?? "notes",
-                    account: hit.account ?? hit.folder_id ?? INTERNAL_NOTES,
-                    ...(hit.url ? { url: hit.url } : {}),
-                    title: hit.title,
-                    timestamp: hit.updated_at,
-                    text: hit.excerpt,
-                    foundBy: hit.found_by,
-                })));
-                const read = chosen?.map(({ account }) => account) ?? [
-                    INTERNAL_NOTES,
-                    ...(await store.notes.folders()).map(({ id }) => id),
-                    ...(await store.notes.documentsAccounts()).map(({ id }) => id),
-                ];
-                accounts.push(...read.map((account) => ({ resource, provider: "notes", account })));
+    // Messages and notes both look up by meaning with the same model: loading it costs most of a second.
+    const embedders = runEmbedders();
+    try {
+        for (const resource of only) {
+            try {
+                if (resource === "notes") {
+                    const chosen = request.scope?.notes;
+                    const found = await searchNotes(store, request.text, {
+                        limit: window,
+                        command,
+                        embedders,
+                        ...(chosen === undefined ? {} : notesScopeOf(chosen)),
+                        ...(request.exact ? { exact: true } : {}),
+                        ...(request.timezone === undefined ? {} : { timezone: request.timezone }),
+                        ...(request.env === undefined ? {} : { env: request.env }),
+                    });
+                    hasMore ||= found.has_more;
+                    notes = {
+                        by: found.by,
+                        ...(found.meaning_skipped === undefined ? {} : { meaningSkipped: found.meaning_skipped }),
+                    };
+                    lists.push(found.hits.map((hit) => ({
+                        kind: hit.kind ?? "note",
+                        ref: hit.ref,
+                        provider: hit.provider ?? "notes",
+                        account: hit.account ?? hit.folder_id ?? INTERNAL_NOTES,
+                        ...(hit.url ? { url: hit.url } : {}),
+                        title: hit.title,
+                        timestamp: hit.updated_at,
+                        text: hit.excerpt,
+                        foundBy: hit.found_by,
+                    })));
+                    const read = chosen?.map(({ account }) => account) ?? [
+                        INTERNAL_NOTES,
+                        ...(await store.notes.folders()).map(({ id }) => id),
+                        ...(await store.notes.documentsAccounts()).map(({ id }) => id),
+                    ];
+                    accounts.push(...read.map((account) => ({ resource, provider: "notes", account })));
+                }
+                else
+                    lists.push(await messagesOf(resource));
+                searched.push(resource);
+            }
+            catch (error) {
+                if (!skippable(error))
+                    throw error;
+                skipped.push({ resource, code: error.code, reason: error.message });
             }
-            else
-                lists.push(await messagesOf(resource));
-            searched.push(resource);
-        }
-        catch (error) {
-            if (!skippable(error))
-                throw error;
-            skipped.push({ resource, code: error.code, reason: error.message });
         }
     }
+    finally {
+        await embedders.close();
+    }
     const ranked = new Map();
     for (const list of lists)
         list.forEach((item, rank) => {
@@ -141,6 +204,7 @@ export const searchAll = async (store, account, request, messenger = {}, searchM
         accounts,
         skipped,
         ...(notes === undefined ? {} : { notes }),
+        ...(messagesBy === undefined ? {} : { messages: messagesBy }),
         ...(server === undefined ? {} : { server }),
     };
 };

package/dist/store/drivers/bun-sqlite.d.ts

@@ -9,6 +9,7 @@ interface BunStatement {
 export interface BunDatabase {
     exec(sql: string): void;
     query(sql: string): BunStatement;
+    loadExtension(path: string): void;
     close(): void;
 }
 export interface BunDatabaseClass {
@@ -25,5 +26,10 @@ export interface BunDatabaseClass {
  */
 export declare const bunDatabase: () => Promise<BunDatabaseClass>;
 export declare const openWithBunSqlite: (path: string) => Promise<CacheDatabase>;
+/** A connection with the vector extension loaded; it closes again when the extension does not load. */
+export declare const bunClient: (path: string, options?: {
+    readonly?: boolean;
+    create?: boolean;
+}) => Promise<BunDatabase>;
 export declare const cacheOverBunSqlite: (database: BunDatabase) => CacheDatabase;
 export {};

package/dist/store/drivers/bun-sqlite.js

@@ -1,3 +1,4 @@
+import { loadVectorExtension, vectorExtension } from "../vector-extension.js";
 let customised = false;
 /**
  * Bun on macOS takes the system's SQLite, which can be older than the store needs, so it always
@@ -29,7 +30,21 @@ export const bunDatabase = async () => {
     }
     return Database;
 };
-export const openWithBunSqlite = async (path) => cacheOverBunSqlite(new (await bunDatabase())(path));
+export const openWithBunSqlite = async (path) => cacheOverBunSqlite(await bunClient(path));
+/** A connection with the vector extension loaded; it closes again when the extension does not load. */
+export const bunClient = async (path, options) => {
+    const Database = await bunDatabase();
+    const extension = await vectorExtension();
+    const client = options ? new Database(path, options) : new Database(path);
+    try {
+        loadVectorExtension(client, extension);
+        return client;
+    }
+    catch (error) {
+        client.close();
+        throw error;
+    }
+};
 export const cacheOverBunSqlite = (database) => {
     return {
         exec: (sql) => database.exec(sql),

package/dist/store/drivers/node-sqlite.d.ts

@@ -4,5 +4,9 @@ import type { CacheDatabase } from "../driver.js";
  * Import this module only through a dynamic import that follows `quietSqliteWarning()`: a static import loads
  * `node:sqlite`, and on Node 22 its ExperimentalWarning, before the filter can run.
  */
-export declare const openWithNodeSqlite: (path: string) => CacheDatabase;
+export declare const openWithNodeSqlite: (path: string) => Promise<CacheDatabase>;
+/** A connection with the vector extension loaded; it closes again when the extension does not load. */
+export declare const nodeClient: (path: string, { readOnly }?: {
+    readOnly?: boolean;
+}) => Promise<DatabaseSync>;
 export declare const cacheOverNodeSqlite: (database: DatabaseSync) => CacheDatabase;

package/dist/store/drivers/node-sqlite.js

@@ -1,10 +1,24 @@
 // biome-ignore-all lint/plugin/no-sqlite-driver: cli-core owns the store, so it opens the driver itself
 import { DatabaseSync } from "node:sqlite";
+import { loadVectorExtension, vectorExtension } from "../vector-extension.js";
 /**
  * Import this module only through a dynamic import that follows `quietSqliteWarning()`: a static import loads
  * `node:sqlite`, and on Node 22 its ExperimentalWarning, before the filter can run.
  */
-export const openWithNodeSqlite = (path) => cacheOverNodeSqlite(new DatabaseSync(path));
+export const openWithNodeSqlite = async (path) => cacheOverNodeSqlite(await nodeClient(path));
+/** A connection with the vector extension loaded; it closes again when the extension does not load. */
+export const nodeClient = async (path, { readOnly = false } = {}) => {
+    const extension = await vectorExtension();
+    const client = new DatabaseSync(path, { readOnly, allowExtension: true });
+    try {
+        loadVectorExtension(client, extension);
+        return client;
+    }
+    catch (error) {
+        client.close();
+        throw error;
+    }
+};
 export const cacheOverNodeSqlite = (database) => {
     return {
         exec: (sql) => database.exec(sql),

package/dist/store/exports.d.ts

@@ -54,6 +54,7 @@ export { savedTaskCategories } from "./sqlite/task-category-settings.js";
 export { DEFAULT_AGENT_CLOSE, savedAgentClose } from "./sqlite/task-close.js";
 export { DEFAULT_FIRST_PASS_PER_CHAT, firstPassPerChatOf, savedFirstPassPerChat } from "./sqlite/task-first-pass.js";
 export { inboxKey } from "./sqlite/tasks.js";
+export { dropAllUnusedVectors, vectorIndexCounts } from "./sqlite/vector-index.js";
 export type { MatchPlace, ScoredHit } from "./sqlite/words.js";
 export type { TaskDetails } from "./sqlite/work-tasks.js";
 export type { SyncStore } from "./sync.js";

package/dist/store/exports.js

@@ -24,3 +24,4 @@ export { savedTaskCategories } from "./sqlite/task-category-settings.js";
 export { DEFAULT_AGENT_CLOSE, savedAgentClose } from "./sqlite/task-close.js";
 export { DEFAULT_FIRST_PASS_PER_CHAT, firstPassPerChatOf, savedFirstPassPerChat } from "./sqlite/task-first-pass.js";
 export { inboxKey } from "./sqlite/tasks.js";
+export { dropAllUnusedVectors, vectorIndexCounts } from "./sqlite/vector-index.js";

package/dist/store/sqlite/accounts.js

@@ -2,6 +2,7 @@ import { CliError } from "@wirecat/cli-base";
 import { and, asc, eq, inArray, sql } from "./drizzle/core.js";
 import { pointerPairs } from "./entity-types.js";
 import { accountIdentities, accounts, aliases, botUpdates, chatEventPeople, chatEvents, chatMembers, chats, chunks, fetchLeases, involvements, memberCounts, memberObservationMembers, memberObservations, memberStays, messageReactions, messageRevisions, messages, messageTranscripts, pollVotes, projects, syncCursors, syncRanges, syncs, } from "./schema.js";
+import { dropUnusedVectors } from "./vector-index.js";
 /** An account's row, made on first sight; a name only replaces what was known when there is one. */
 export const accountPk = ({ orm, now }, { provider, account, scope }, name = null) => Number(orm
     .insert(accounts)
@@ -114,8 +115,7 @@ export const purgeAccount = ({ orm, database }, accountKey) => {
     orm.run(sql `DELETE FROM files WHERE account_id = ${accountKey}`);
     orm.delete(accounts).where(eq(accounts.id, accountKey)).run();
     dropOrphanPeople(database, unseenBefore);
-    orm.run(sql `DELETE FROM embeddings WHERE content_hash IN (SELECT value FROM json_each(${JSON.stringify(hashes)}))
-      AND NOT EXISTS (SELECT 1 FROM chunks k WHERE k.content_hash = embeddings.content_hash)`);
+    dropUnusedVectors(database, hashes);
 };
 /** Identity and person rows a person's own record keeps, not evidence that anyone saw them. */
 const OWN = new Set(["identity_links", "identity_link_events", "identity_revisions"]);

package/dist/store/sqlite/chats.d.ts

@@ -60,7 +60,7 @@ export declare const leftChats: ({ orm }: StoreContext, accountKey: number) => {
     messages: number;
 };
 /** These chats and everything under them, children before parents: the foreign keys are enforced. */
-export declare const purgeChats: ({ orm }: StoreContext, pks: number[]) => void;
+export declare const purgeChats: ({ orm, database }: StoreContext, pks: number[]) => void;
 export declare const chatsWith: ({ orm, now }: StoreContext, accountKey: number, key: AccountKey, memberId: Id) => Chat[];
 /**
  * How the store answers a chat list's query. `title:` goes through the trigram index, so it matches part of a

package/dist/store/sqlite/chats.js

@@ -5,6 +5,7 @@ import { bound } from "./query-filter.js";
 import { chatEventPeople, chatEvents, chatMembers, chats, chunks, fetchLeases, identities, involvements, memberCounts, memberObservationMembers, memberObservations, memberStays, messageReactions, messageRevisions, messages, messageTranscripts, pollVotes, syncRanges, } from "./schema.js";
 import { ensureTag } from "./tags.js";
 import { parsed, present, toIso, toMs } from "./values.js";
+import { dropUnusedVectors } from "./vector-index.js";
 /** The chat's own id, when the store key is a chat of this account. */
 export const chatOf = ({ orm }, accountKey, chatKey) => orm
     .select({ id: chats.externalId })
@@ -266,14 +267,19 @@ export const leftChats = ({ orm }, accountKey) => {
     return { pks, chats: left, messages: left.reduce((sum, chat) => sum + chat.messages, 0) };
 };
 /** These chats and everything under them, children before parents: the foreign keys are enforced. */
-export const purgeChats = ({ orm }, pks) => {
+export const purgeChats = ({ orm, database }, pks) => {
     if (pks.length === 0)
         return;
     const messagesOf = orm.select({ pk: messages.id }).from(messages).where(inArray(messages.chatId, pks));
-    orm
+    const chatPks = sql `(SELECT value FROM json_each(${JSON.stringify(pks)}))`;
+    const released = orm
         .delete(chunks)
-        .where(sql `${chunks.chunkableType}='conversation' AND ${chunks.chunkableId} IN (SELECT id FROM conversations WHERE chat_id IN (SELECT value FROM json_each(${JSON.stringify(pks)})))`)
-        .run();
+        .where(sql `(${chunks.chunkableType}='conversation' AND ${chunks.chunkableId} IN (SELECT id FROM conversations WHERE chat_id IN ${chatPks}))
+        OR (${chunks.chunkableType}='message' AND ${chunks.chunkableId} IN (SELECT id FROM messages WHERE chat_id IN ${chatPks}))
+        OR (${chunks.chunkableType}='image' AND ${chunks.chunkableId} IN (SELECT a.id FROM attachments a
+          JOIN messages m ON a.attachable_type = 'message' AND m.id = a.attachable_id WHERE m.chat_id IN ${chatPks}))`)
+        .returning({ hash: chunks.contentHash })
+        .all();
     orm
         .delete(involvements)
         .where(sql `(${involvements.subjectType}='chat' AND ${involvements.subjectId} IN (SELECT value FROM json_each(${JSON.stringify(pks)}))) OR (${involvements.subjectType}='message' AND ${involvements.subjectId} IN (${messagesOf}))`)
@@ -303,6 +309,7 @@ export const purgeChats = ({ orm }, pks) => {
     orm.delete(messageRevisions).where(inArray(messageRevisions.messageId, messagesOf)).run();
     orm.delete(messages).where(inArray(messages.chatId, pks)).run();
     orm.delete(chats).where(inArray(chats.id, pks)).run();
+    dropUnusedVectors(database, released.map(({ hash }) => hash));
 };
 export const chatsWith = ({ orm, now }, accountKey, key, memberId) => {
     const rows = orm

package/dist/store/sqlite/document-pictures.js

@@ -1,5 +1,6 @@
 import { CliError } from "@wirecat/cli-base";
 import { write } from "./transaction.js";
+import { dropUnusedVectors, ensureVectorIndex, unitVector } from "./vector-index.js";
 import { dot } from "./vectors.js";
 /** A document's picture is one chunk of this type at position 0; its vector is the picture's, not text's. */
 export const DOCUMENT_PICTURE = "document_picture";
@@ -10,11 +11,6 @@ const idOf = (ref) => {
         throw new CliError("validation_error", `${ref} is not a document reference (document:<id>)`);
     return Number(match[1]);
 };
-const purge = (database, hashes) => {
-    const drop = database.prepare("DELETE FROM embeddings WHERE content_hash = ? AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ?)");
-    for (const hash of hashes)
-        drop.run(hash, hash);
-};
 /** Drops a document's picture chunk, and its vector when no other chunk shares the bytes. */
 export const dropDocumentPicture = (database, documentId) => {
     const hashes = database
@@ -22,7 +18,7 @@ export const dropDocumentPicture = (database, documentId) => {
         .all(DOCUMENT_PICTURE, documentId)
         .map((row) => String(row.hash));
     database.prepare("DELETE FROM chunks WHERE chunkable_type = ? AND chunkable_id = ?").run(DOCUMENT_PICTURE, documentId);
-    purge(database, hashes);
+    dropUnusedVectors(database, hashes);
     return hashes.length > 0;
 };
 export const documentPicturesOver = ({ database, now }) => ({
@@ -44,10 +40,12 @@ export const documentPicturesOver = ({ database, now }) => ({
                 throw new CliError("not_found", `no document ${ref} in the local store`);
             const at = now();
             dropDocumentPicture(database, id);
+            ensureVectorIndex(database, model, dims);
+            const unit = unitVector(vector);
             database
                 .prepare(`INSERT INTO embeddings (model, content_hash, dims, vector, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)
             ON CONFLICT DO NOTHING`)
-                .run(model, hash, dims, Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength), at, at);
+                .run(model, hash, dims, Buffer.from(unit.buffer, unit.byteOffset, unit.byteLength), at, at);
             database
                 .prepare(`INSERT INTO chunks (chunkable_type, chunkable_id, position, start_offset, end_offset, content_hash, scope,
               account_id, occurred_at, created_at, updated_at) VALUES (?, ?, 0, 0, ?, ?, ?, ?, ?, ?, ?)`)

package/dist/store/sqlite/email-index.d.ts

@@ -13,10 +13,8 @@ export declare const MAIL_PIECES = "(SELECT k.id, k.content_hash, k.chunkable_ty
  */
 export declare const threadPieces: (database: CacheDatabase, threadId: number) => Chunk[];
 export declare const queueThreads: (database: CacheDatabase, ids: Iterable<number>) => void;
-/** Drops the pieces of an email's attachment texts; answers their hashes, for `purgeUnused` once new ones are in. */
+/** Drops the pieces of an email's attachment texts; answers their hashes, for `dropUnusedVectors` once new ones are in. */
 export declare const dropAttachmentChunks: (database: CacheDatabase, emailId: number) => string[];
-/** Vectors are keyed by text alone, so one goes only when no piece of anything holds that text any more. */
-export declare const purgeUnused: (database: CacheDatabase, hashes: string[]) => void;
 /** Each attachment's extracted text, in pieces for search by meaning, as an email's own text is. */
 export declare const chunkAttachments: (database: CacheDatabase, accountId: number, emailId: number, now: number) => void;
 export declare const drainEmailIndex: (database: CacheDatabase, stemmerFor: (stemmers: Stemmers) => Stemmer) => number;

package/dist/store/sqlite/email-index.js

@@ -3,6 +3,7 @@ import { mailCore } from "../../conversations/mail-text.js";
 import { int, str } from "./events.js";
 import { CORPORA, drainCorpus } from "./note-index.js";
 import { write } from "./transaction.js";
+import { dropUnusedVectors } from "./vector-index.js";
 export const titleOf = (subject) => subject?.replace(/^\s*((re|fwd?|aw|wg)\s*:\s*)+/i, "").trim() || null;
 /**
  * Every piece of mail: a stretch of a thread, or of one email's attachment. A thread piece holds no email of its
@@ -58,7 +59,7 @@ const chunkThread = (database, threadId, now) => {
         for (const [position, piece] of cutThread(lines).entries())
             insert.run(threadId, position, piece.text.length, piece.hash, str(thread.scope), Number(thread.account_id), at.get(piece.lastId) ?? null, now, now);
     }
-    purgeUnused(database, before);
+    dropUnusedVectors(database, before);
 };
 export const queueThreads = (database, ids) => {
     const queue = database.prepare("INSERT OR IGNORE INTO email_index_pending (indexable_type, id) VALUES ('email_thread', ?)");
@@ -66,7 +67,7 @@ export const queueThreads = (database, ids) => {
         queue.run(id);
 };
 const ATTACHMENTS_OF_EMAIL = "SELECT id FROM attachments WHERE attachable_type = 'email' AND attachable_id = ?";
-/** Drops the pieces of an email's attachment texts; answers their hashes, for `purgeUnused` once new ones are in. */
+/** Drops the pieces of an email's attachment texts; answers their hashes, for `dropUnusedVectors` once new ones are in. */
 export const dropAttachmentChunks = (database, emailId) => {
     const hashes = database
         .prepare(`SELECT content_hash AS hash FROM chunks WHERE chunkable_type = 'attachment' AND chunkable_id IN (${ATTACHMENTS_OF_EMAIL})`)
@@ -77,23 +78,6 @@ export const dropAttachmentChunks = (database, emailId) => {
         .run(emailId);
     return hashes;
 };
-/** Vectors are keyed by text alone, so one goes only when no piece of anything holds that text any more. */
-export const purgeUnused = (database, hashes) => {
-    const unique = new Set(hashes);
-    if (unique.size === 0)
-        return;
-    // The key is (model, content_hash): naming the model reaches the row by the key; the hash alone scans every vector.
-    const models = database
-        .prepare(`WITH RECURSIVE m(model) AS (SELECT min(model) FROM embeddings
-         UNION ALL SELECT (SELECT min(model) FROM embeddings WHERE model > m.model) FROM m WHERE m.model IS NOT NULL)
-       SELECT model FROM m WHERE model IS NOT NULL`)
-        .all()
-        .map(({ model }) => String(model));
-    const purge = database.prepare("DELETE FROM embeddings WHERE model = ? AND content_hash = ? AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ?)");
-    for (const hash of unique)
-        for (const model of models)
-            purge.run(model, hash, hash);
-};
 /** Each attachment's extracted text, in pieces for search by meaning, as an email's own text is. */
 export const chunkAttachments = (database, accountId, emailId, now) => {
     const email = database
@@ -125,7 +109,7 @@ export const drainEmailIndex = (database, stemmerFor) => {
                 .prepare("SELECT id FROM email_threads")
                 .all()
                 .map(({ id }) => Number(id)));
-            purgeUnused(database, single);
+            dropUnusedVectors(database, single);
             database
                 .prepare("INSERT OR REPLACE INTO store_settings (key, value, at) VALUES (?, ?, ?)")
                 .run(MAIL_CUT, CUT, Date.now());
@@ -146,7 +130,7 @@ export const drainEmailIndex = (database, stemmerFor) => {
             for (const row of unchunked) {
                 const before = dropAttachmentChunks(database, Number(row.id));
                 chunkAttachments(database, Number(row.account_id), Number(row.id), Date.now());
-                purgeUnused(database, before);
+                dropUnusedVectors(database, before);
             }
         });
     const emails = drainCorpus(database, CORPORA.email, stemmerFor);

package/dist/store/sqlite/emails.js

@@ -1,7 +1,7 @@
 import { formatLocator } from "../../domain/locator.js";
 import { formatReference } from "../../domain/references.js";
 import { normalize } from "../normalize.js";
-import { chunkAttachments, drainEmailIndex, dropAttachmentChunks, MAIL_PIECES, purgeUnused, queueThreads, threadPieces, titleOf, } from "./email-index.js";
+import { chunkAttachments, drainEmailIndex, dropAttachmentChunks, MAIL_PIECES, queueThreads, threadPieces, titleOf, } from "./email-index.js";
 import { recountEmailThread as recount, setEmailFlags } from "./email-state.js";
 import { bool, flag, fromJson, int, page, str, toJson } from "./events.js";
 import { ensurePerson } from "./identities.js";
@@ -9,6 +9,7 @@ import { mailRulesOver } from "./mail-rules.js";
 import { attachmentOf, saveAttachment, wordsQuery } from "./meetings.js";
 import { stemmerCache } from "./stems.js";
 import { write } from "./transaction.js";
+import { dropUnusedVectors } from "./vector-index.js";
 import { dot } from "./vectors.js";
 /** A thread as a conversation: the summary `search conversations` and `related` answer for an email. */
 export const threadConversation = (place, builtAt) => ({
@@ -234,7 +235,7 @@ const saveEmail = (database, orm, accountId, threadId, mail, now) => {
         for (const a of mail.attachments)
             saveAttachment(database, "email", emailId, { ...a, normalizedText: a.text === null ? null : normalize(a.text) }, now);
         chunkAttachments(database, accountId, emailId, now);
-        purgeUnused(database, before);
+        dropUnusedVectors(database, before);
     }
     return found ? Number(found.email_thread_id) : null;
 };
@@ -547,7 +548,7 @@ export const mailStoreOver = ({ database, orm }, stemmerFor = stemmerCache()) =>
             const threads = externalIds.flatMap((id) => mark.all(now, now, accountId, id).map((r) => {
                 const pieces = dropAttachmentChunks(database, Number(r.id));
                 forget.run(Number(r.id));
-                purgeUnused(database, pieces);
+                dropUnusedVectors(database, pieces);
                 return Number(r.email_thread_id);
             }));
             for (const id of new Set(threads))

package/dist/store/sqlite/image-chunks.d.ts

@@ -34,7 +34,7 @@ export interface ImageChunk {
  * Makes these the chunks of their images, and drops the chunks of images that are gone or whose message was
  * deleted, with the vectors nothing else uses any more.
  */
-export declare const saveImageChunks: ({ orm, now }: StoreContext, accountPk: number, chatKey: number, chunks: ImageChunk[], keep: number[]) => void;
+export declare const saveImageChunks: ({ orm, now, database }: StoreContext, accountPk: number, chatKey: number, chunks: ImageChunk[], keep: number[]) => void;
 /** The hashes among these that already have a vector of `model`. */
 export declare const embeddedHashes: ({ orm }: StoreContext, model: string, hashes: string[]) => Set<string>;
 export interface NearestImage {
@@ -46,7 +46,7 @@ export interface NearestImage {
  * The images of the account nearest to `query` by the vector of `model`, best first. Position 0 of an image is
  * its context text, position 1 its picture (`PICTURE_POSITION`): a model's vectors only ever sit at one.
  */
-export declare const nearestImages: ({ orm }: StoreContext, accountPk: number, { chatKey, model, since, limit, query, position, }: {
+export declare const nearestImages: (context: StoreContext, accountPk: number, { chatKey, model, since, limit, query, position, }: {
     chatKey?: number;
     model: string;
     since?: number;
@@ -75,7 +75,7 @@ export declare const picturesToEmbed: ({ orm }: StoreContext, accountPk: number,
     limit: number;
 }) => PictureInput[];
 /** Makes each hash the picture chunk of its image; the vectors are saved apart, by `saveVectors`. */
-export declare const savePictureChunks: ({ orm, now }: StoreContext, accountPk: number, pictures: {
+export declare const savePictureChunks: ({ orm, now, database }: StoreContext, accountPk: number, pictures: {
     attachmentPk: number;
     hash: string;
     bytes: number;