@wirecat/cli-core 0.434.0 → 0.435.0

package/dist/services/search-all.js

@@ -1,18 +1,38 @@
 import { CliError } from "@wirecat/cli-base";
+import { parseLucene } from "@wirecat/cli-base/query";
 import { parseLocator } from "../domain/locator.js";
 import { INTERNAL_NOTES, kindIn } from "../domain/providers.js";
+import { runEmbedders } from "../embeddings/embed.js";
+import { DEFAULT_TEXT_MODEL } from "../embeddings/models.js";
+import { MESSAGE_PARSING } from "../search/lucene/types.js";
+import { planQuestion } from "../search/question-plan.js";
 import { excerpt } from "./excerpt.js";
 import { MAX_SEARCH_OFFSET, searchStore } from "./messages-archive-search.js";
+import { nearEligible } from "./messages-discovery.js";
 import { notesScopeOf, RRF_K, searchNotes } from "./notes-search.js";
 export const RESOURCES_SEARCHED = ["messages", "mail", "notes"];
 /** A query a resource cannot answer is skipped with its reason, never a failure of the whole search. */
 const skippable = (error) => error instanceof CliError && (error.code === "validation_error" || error.code === "not_found");
+/**
+ * The filters the meaning hits must pass, or `undefined` when they cannot be followed: a NOT anywhere (meaning would
+ * bring the excluded back), or a filter under OR or in a group, which the plain list of filters would turn into AND.
+ */
+const filtersBeside = (text) => {
+    const { root } = parseLucene(text, MESSAGE_PARSING);
+    const followable = (node, required) => node.kind === "boolean"
+        ? node.clauses.every(({ node: inner, occur }) => occur !== "must_not" && followable(inner, false))
+        : node.kind === "predicate" && (required || node.field === "text" || node.field === "exact");
+    const top = root.kind === "boolean" ? root.clauses : [{ node: root, occur: "must" }];
+    return top.every(({ node, occur }) => occur !== "must_not" && followable(node, occur === "must"))
+        ? planQuestion(text).scope
+        : undefined;
+};
 /**
  * `search all`: messages, mail and notes from the local store — messages also from the messenger's
  * server, as `search messages` asks it — each list best first, merged by reciprocal rank so no
  * resource's own scores have to be compared with another's.
  */
-export const searchAll = async (store, account, request, messenger = {}, searchMessages = (query) => searchStore(store, account, query, messenger)) => {
+export const searchAll = async (store, account, request, messenger = {}, searchMessages = (query) => searchStore(store, account, query, messenger), meaning) => {
     if (!request.text.trim())
         throw new CliError("validation_error", "say what to find");
     const offset = request.offset ?? 0;
@@ -28,6 +48,7 @@ export const searchAll = async (store, account, request, messenger = {}, searchM
     const skipped = [];
     let hasMore = false;
     let notes;
+    let messagesBy;
     let server;
     const messagesOf = async (kind) => {
         const given = request.scope?.[kind];
@@ -60,7 +81,7 @@ export const searchAll = async (store, account, request, messenger = {}, searchM
             if (!skipped.some(({ source }) => source === one.source))
                 skipped.push(one);
         accounts.push(...read.map(({ provider, account }) => ({ resource: kind, provider, account })));
-        return found.items.map((hit) => {
+        const itemOf = (hit, foundBy) => {
             const at = parseLocator(hit.locator);
             return {
                 kind: kind === "mail" ? "mail" : "message",
@@ -74,55 +95,97 @@ export const searchAll = async (store, account, request, messenger = {}, searchM
                     ? { name: hit.email.from.name ?? hit.email.from.address, ref: hit.email.from.ref }
                     : { name: hit.senderName, ref: null },
                 ...(hit.email ? { email: hit.email } : {}),
-                foundBy: ["words"],
+                foundBy,
             };
+        };
+        const byWords = found.items.map((hit) => itemOf(hit, ["words"]));
+        // Server hits are only those the server returned; the local vectors would add others.
+        if (kind === "mail" || !meaning || request.exact || request.backend === "server")
+            return byWords;
+        const filters = filtersBeside(request.text);
+        const lane = filters === undefined
+            ? {
+                skipped: {
+                    reason: "the query uses NOT, or a filter under OR or in a group, which meaning cannot follow",
+                    model: DEFAULT_TEXT_MODEL,
+                },
+            }
+            : await meaning(request.text, read, embedders);
+        if ("skipped" in lane) {
+            messagesBy = { by: "words", meaningSkipped: lane.skipped };
+            return byWords;
+        }
+        messagesBy = { by: "words and meaning", ...(lane.partly ? { meaningSkipped: lane.partly } : {}) };
+        const near = await nearEligible(store, account, query, messenger, lane.ranked, filters ?? "");
+        // One list per resource, as notes give theirs, so messages get no second vote against mail.
+        const fused = new Map();
+        byWords.forEach((item, rank) => {
+            fused.set(item.ref, { item, score: 1 / (RRF_K + rank + 1) });
         });
+        near.forEach((hit, rank) => {
+            const held = fused.get(hit.locator);
+            if (held)
+                held.item.foundBy = ["words", "meaning"];
+            fused.set(hit.locator, {
+                item: held?.item ?? itemOf(hit, ["meaning"]),
+                score: (held?.score ?? 0) + 1 / (RRF_K + rank + 1),
+            });
+        });
+        return [...fused.values()].sort((a, b) => b.score - a.score).map(({ item }) => item);
     };
-    for (const resource of only) {
-        try {
-            if (resource === "notes") {
-                const chosen = request.scope?.notes;
-                const found = await searchNotes(store, request.text, {
-                    limit: window,
-                    command,
-                    ...(chosen === undefined ? {} : notesScopeOf(chosen)),
-                    ...(request.exact ? { exact: true } : {}),
-                    ...(request.timezone === undefined ? {} : { timezone: request.timezone }),
-                    ...(request.env === undefined ? {} : { env: request.env }),
-                });
-                hasMore ||= found.has_more;
-                notes = {
-                    by: found.by,
-                    ...(found.meaning_skipped === undefined ? {} : { meaningSkipped: found.meaning_skipped }),
-                };
-                lists.push(found.hits.map((hit) => ({
-                    kind: hit.kind ?? "note",
-                    ref: hit.ref,
-                    provider: hit.provider ?? "notes",
-                    account: hit.account ?? hit.folder_id ?? INTERNAL_NOTES,
-                    ...(hit.url ? { url: hit.url } : {}),
-                    title: hit.title,
-                    timestamp: hit.updated_at,
-                    text: hit.excerpt,
-                    foundBy: hit.found_by,
-                })));
-                const read = chosen?.map(({ account }) => account) ?? [
-                    INTERNAL_NOTES,
-                    ...(await store.notes.folders()).map(({ id }) => id),
-                    ...(await store.notes.documentsAccounts()).map(({ id }) => id),
-                ];
-                accounts.push(...read.map((account) => ({ resource, provider: "notes", account })));
+    // Messages and notes both look up by meaning with the same model: loading it costs most of a second.
+    const embedders = runEmbedders();
+    try {
+        for (const resource of only) {
+            try {
+                if (resource === "notes") {
+                    const chosen = request.scope?.notes;
+                    const found = await searchNotes(store, request.text, {
+                        limit: window,
+                        command,
+                        embedders,
+                        ...(chosen === undefined ? {} : notesScopeOf(chosen)),
+                        ...(request.exact ? { exact: true } : {}),
+                        ...(request.timezone === undefined ? {} : { timezone: request.timezone }),
+                        ...(request.env === undefined ? {} : { env: request.env }),
+                    });
+                    hasMore ||= found.has_more;
+                    notes = {
+                        by: found.by,
+                        ...(found.meaning_skipped === undefined ? {} : { meaningSkipped: found.meaning_skipped }),
+                    };
+                    lists.push(found.hits.map((hit) => ({
+                        kind: hit.kind ?? "note",
+                        ref: hit.ref,
+                        provider: hit.provider ?? "notes",
+                        account: hit.account ?? hit.folder_id ?? INTERNAL_NOTES,
+                        ...(hit.url ? { url: hit.url } : {}),
+                        title: hit.title,
+                        timestamp: hit.updated_at,
+                        text: hit.excerpt,
+                        foundBy: hit.found_by,
+                    })));
+                    const read = chosen?.map(({ account }) => account) ?? [
+                        INTERNAL_NOTES,
+                        ...(await store.notes.folders()).map(({ id }) => id),
+                        ...(await store.notes.documentsAccounts()).map(({ id }) => id),
+                    ];
+                    accounts.push(...read.map((account) => ({ resource, provider: "notes", account })));
+                }
+                else
+                    lists.push(await messagesOf(resource));
+                searched.push(resource);
+            }
+            catch (error) {
+                if (!skippable(error))
+                    throw error;
+                skipped.push({ resource, code: error.code, reason: error.message });
             }
-            else
-                lists.push(await messagesOf(resource));
-            searched.push(resource);
-        }
-        catch (error) {
-            if (!skippable(error))
-                throw error;
-            skipped.push({ resource, code: error.code, reason: error.message });
         }
     }
+    finally {
+        await embedders.close();
+    }
     const ranked = new Map();
     for (const list of lists)
         list.forEach((item, rank) => {
@@ -141,6 +204,7 @@ export const searchAll = async (store, account, request, messenger = {}, searchM
         accounts,
         skipped,
         ...(notes === undefined ? {} : { notes }),
+        ...(messagesBy === undefined ? {} : { messages: messagesBy }),
         ...(server === undefined ? {} : { server }),
     };
 };

package/dist/store/exports.d.ts

@@ -54,7 +54,7 @@ export { savedTaskCategories } from "./sqlite/task-category-settings.js";
 export { DEFAULT_AGENT_CLOSE, savedAgentClose } from "./sqlite/task-close.js";
 export { DEFAULT_FIRST_PASS_PER_CHAT, firstPassPerChatOf, savedFirstPassPerChat } from "./sqlite/task-first-pass.js";
 export { inboxKey } from "./sqlite/tasks.js";
-export { vectorIndexCounts } from "./sqlite/vector-index.js";
+export { dropAllUnusedVectors, vectorIndexCounts } from "./sqlite/vector-index.js";
 export type { MatchPlace, ScoredHit } from "./sqlite/words.js";
 export type { TaskDetails } from "./sqlite/work-tasks.js";
 export type { SyncStore } from "./sync.js";

package/dist/store/exports.js

@@ -24,4 +24,4 @@ export { savedTaskCategories } from "./sqlite/task-category-settings.js";
 export { DEFAULT_AGENT_CLOSE, savedAgentClose } from "./sqlite/task-close.js";
 export { DEFAULT_FIRST_PASS_PER_CHAT, firstPassPerChatOf, savedFirstPassPerChat } from "./sqlite/task-first-pass.js";
 export { inboxKey } from "./sqlite/tasks.js";
-export { vectorIndexCounts } from "./sqlite/vector-index.js";
+export { dropAllUnusedVectors, vectorIndexCounts } from "./sqlite/vector-index.js";

package/dist/store/sqlite/accounts.js

@@ -2,6 +2,7 @@ import { CliError } from "@wirecat/cli-base";
 import { and, asc, eq, inArray, sql } from "./drizzle/core.js";
 import { pointerPairs } from "./entity-types.js";
 import { accountIdentities, accounts, aliases, botUpdates, chatEventPeople, chatEvents, chatMembers, chats, chunks, fetchLeases, involvements, memberCounts, memberObservationMembers, memberObservations, memberStays, messageReactions, messageRevisions, messages, messageTranscripts, pollVotes, projects, syncCursors, syncRanges, syncs, } from "./schema.js";
+import { dropUnusedVectors } from "./vector-index.js";
 /** An account's row, made on first sight; a name only replaces what was known when there is one. */
 export const accountPk = ({ orm, now }, { provider, account, scope }, name = null) => Number(orm
     .insert(accounts)
@@ -114,8 +115,7 @@ export const purgeAccount = ({ orm, database }, accountKey) => {
     orm.run(sql `DELETE FROM files WHERE account_id = ${accountKey}`);
     orm.delete(accounts).where(eq(accounts.id, accountKey)).run();
     dropOrphanPeople(database, unseenBefore);
-    orm.run(sql `DELETE FROM embeddings WHERE content_hash IN (SELECT value FROM json_each(${JSON.stringify(hashes)}))
-      AND NOT EXISTS (SELECT 1 FROM chunks k WHERE k.content_hash = embeddings.content_hash)`);
+    dropUnusedVectors(database, hashes);
 };
 /** Identity and person rows a person's own record keeps, not evidence that anyone saw them. */
 const OWN = new Set(["identity_links", "identity_link_events", "identity_revisions"]);

package/dist/store/sqlite/chats.d.ts

@@ -60,7 +60,7 @@ export declare const leftChats: ({ orm }: StoreContext, accountKey: number) => {
     messages: number;
 };
 /** These chats and everything under them, children before parents: the foreign keys are enforced. */
-export declare const purgeChats: ({ orm }: StoreContext, pks: number[]) => void;
+export declare const purgeChats: ({ orm, database }: StoreContext, pks: number[]) => void;
 export declare const chatsWith: ({ orm, now }: StoreContext, accountKey: number, key: AccountKey, memberId: Id) => Chat[];
 /**
  * How the store answers a chat list's query. `title:` goes through the trigram index, so it matches part of a

package/dist/store/sqlite/chats.js

@@ -5,6 +5,7 @@ import { bound } from "./query-filter.js";
 import { chatEventPeople, chatEvents, chatMembers, chats, chunks, fetchLeases, identities, involvements, memberCounts, memberObservationMembers, memberObservations, memberStays, messageReactions, messageRevisions, messages, messageTranscripts, pollVotes, syncRanges, } from "./schema.js";
 import { ensureTag } from "./tags.js";
 import { parsed, present, toIso, toMs } from "./values.js";
+import { dropUnusedVectors } from "./vector-index.js";
 /** The chat's own id, when the store key is a chat of this account. */
 export const chatOf = ({ orm }, accountKey, chatKey) => orm
     .select({ id: chats.externalId })
@@ -266,14 +267,19 @@ export const leftChats = ({ orm }, accountKey) => {
     return { pks, chats: left, messages: left.reduce((sum, chat) => sum + chat.messages, 0) };
 };
 /** These chats and everything under them, children before parents: the foreign keys are enforced. */
-export const purgeChats = ({ orm }, pks) => {
+export const purgeChats = ({ orm, database }, pks) => {
     if (pks.length === 0)
         return;
     const messagesOf = orm.select({ pk: messages.id }).from(messages).where(inArray(messages.chatId, pks));
-    orm
+    const chatPks = sql `(SELECT value FROM json_each(${JSON.stringify(pks)}))`;
+    const released = orm
         .delete(chunks)
-        .where(sql `${chunks.chunkableType}='conversation' AND ${chunks.chunkableId} IN (SELECT id FROM conversations WHERE chat_id IN (SELECT value FROM json_each(${JSON.stringify(pks)})))`)
-        .run();
+        .where(sql `(${chunks.chunkableType}='conversation' AND ${chunks.chunkableId} IN (SELECT id FROM conversations WHERE chat_id IN ${chatPks}))
+        OR (${chunks.chunkableType}='message' AND ${chunks.chunkableId} IN (SELECT id FROM messages WHERE chat_id IN ${chatPks}))
+        OR (${chunks.chunkableType}='image' AND ${chunks.chunkableId} IN (SELECT a.id FROM attachments a
+          JOIN messages m ON a.attachable_type = 'message' AND m.id = a.attachable_id WHERE m.chat_id IN ${chatPks}))`)
+        .returning({ hash: chunks.contentHash })
+        .all();
     orm
         .delete(involvements)
         .where(sql `(${involvements.subjectType}='chat' AND ${involvements.subjectId} IN (SELECT value FROM json_each(${JSON.stringify(pks)}))) OR (${involvements.subjectType}='message' AND ${involvements.subjectId} IN (${messagesOf}))`)
@@ -303,6 +309,7 @@ export const purgeChats = ({ orm }, pks) => {
     orm.delete(messageRevisions).where(inArray(messageRevisions.messageId, messagesOf)).run();
     orm.delete(messages).where(inArray(messages.chatId, pks)).run();
     orm.delete(chats).where(inArray(chats.id, pks)).run();
+    dropUnusedVectors(database, released.map(({ hash }) => hash));
 };
 export const chatsWith = ({ orm, now }, accountKey, key, memberId) => {
     const rows = orm

package/dist/store/sqlite/document-pictures.js

@@ -1,6 +1,6 @@
 import { CliError } from "@wirecat/cli-base";
 import { write } from "./transaction.js";
-import { ensureVectorIndex, unitVector } from "./vector-index.js";
+import { dropUnusedVectors, ensureVectorIndex, unitVector } from "./vector-index.js";
 import { dot } from "./vectors.js";
 /** A document's picture is one chunk of this type at position 0; its vector is the picture's, not text's. */
 export const DOCUMENT_PICTURE = "document_picture";
@@ -11,11 +11,6 @@ const idOf = (ref) => {
         throw new CliError("validation_error", `${ref} is not a document reference (document:<id>)`);
     return Number(match[1]);
 };
-const purge = (database, hashes) => {
-    const drop = database.prepare("DELETE FROM embeddings WHERE content_hash = ? AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ?)");
-    for (const hash of hashes)
-        drop.run(hash, hash);
-};
 /** Drops a document's picture chunk, and its vector when no other chunk shares the bytes. */
 export const dropDocumentPicture = (database, documentId) => {
     const hashes = database
@@ -23,7 +18,7 @@ export const dropDocumentPicture = (database, documentId) => {
         .all(DOCUMENT_PICTURE, documentId)
         .map((row) => String(row.hash));
     database.prepare("DELETE FROM chunks WHERE chunkable_type = ? AND chunkable_id = ?").run(DOCUMENT_PICTURE, documentId);
-    purge(database, hashes);
+    dropUnusedVectors(database, hashes);
     return hashes.length > 0;
 };
 export const documentPicturesOver = ({ database, now }) => ({

package/dist/store/sqlite/email-index.d.ts

@@ -13,10 +13,8 @@ export declare const MAIL_PIECES = "(SELECT k.id, k.content_hash, k.chunkable_ty
  */
 export declare const threadPieces: (database: CacheDatabase, threadId: number) => Chunk[];
 export declare const queueThreads: (database: CacheDatabase, ids: Iterable<number>) => void;
-/** Drops the pieces of an email's attachment texts; answers their hashes, for `purgeUnused` once new ones are in. */
+/** Drops the pieces of an email's attachment texts; answers their hashes, for `dropUnusedVectors` once new ones are in. */
 export declare const dropAttachmentChunks: (database: CacheDatabase, emailId: number) => string[];
-/** Vectors are keyed by text alone, so one goes only when no piece of anything holds that text any more. */
-export declare const purgeUnused: (database: CacheDatabase, hashes: string[]) => void;
 /** Each attachment's extracted text, in pieces for search by meaning, as an email's own text is. */
 export declare const chunkAttachments: (database: CacheDatabase, accountId: number, emailId: number, now: number) => void;
 export declare const drainEmailIndex: (database: CacheDatabase, stemmerFor: (stemmers: Stemmers) => Stemmer) => number;

package/dist/store/sqlite/email-index.js

@@ -3,6 +3,7 @@ import { mailCore } from "../../conversations/mail-text.js";
 import { int, str } from "./events.js";
 import { CORPORA, drainCorpus } from "./note-index.js";
 import { write } from "./transaction.js";
+import { dropUnusedVectors } from "./vector-index.js";
 export const titleOf = (subject) => subject?.replace(/^\s*((re|fwd?|aw|wg)\s*:\s*)+/i, "").trim() || null;
 /**
  * Every piece of mail: a stretch of a thread, or of one email's attachment. A thread piece holds no email of its
@@ -58,7 +59,7 @@ const chunkThread = (database, threadId, now) => {
         for (const [position, piece] of cutThread(lines).entries())
             insert.run(threadId, position, piece.text.length, piece.hash, str(thread.scope), Number(thread.account_id), at.get(piece.lastId) ?? null, now, now);
     }
-    purgeUnused(database, before);
+    dropUnusedVectors(database, before);
 };
 export const queueThreads = (database, ids) => {
     const queue = database.prepare("INSERT OR IGNORE INTO email_index_pending (indexable_type, id) VALUES ('email_thread', ?)");
@@ -66,7 +67,7 @@ export const queueThreads = (database, ids) => {
         queue.run(id);
 };
 const ATTACHMENTS_OF_EMAIL = "SELECT id FROM attachments WHERE attachable_type = 'email' AND attachable_id = ?";
-/** Drops the pieces of an email's attachment texts; answers their hashes, for `purgeUnused` once new ones are in. */
+/** Drops the pieces of an email's attachment texts; answers their hashes, for `dropUnusedVectors` once new ones are in. */
 export const dropAttachmentChunks = (database, emailId) => {
     const hashes = database
         .prepare(`SELECT content_hash AS hash FROM chunks WHERE chunkable_type = 'attachment' AND chunkable_id IN (${ATTACHMENTS_OF_EMAIL})`)
@@ -77,23 +78,6 @@ export const dropAttachmentChunks = (database, emailId) => {
         .run(emailId);
     return hashes;
 };
-/** Vectors are keyed by text alone, so one goes only when no piece of anything holds that text any more. */
-export const purgeUnused = (database, hashes) => {
-    const unique = new Set(hashes);
-    if (unique.size === 0)
-        return;
-    // The key is (model, content_hash): naming the model reaches the row by the key; the hash alone scans every vector.
-    const models = database
-        .prepare(`WITH RECURSIVE m(model) AS (SELECT min(model) FROM embeddings
-         UNION ALL SELECT (SELECT min(model) FROM embeddings WHERE model > m.model) FROM m WHERE m.model IS NOT NULL)
-       SELECT model FROM m WHERE model IS NOT NULL`)
-        .all()
-        .map(({ model }) => String(model));
-    const purge = database.prepare("DELETE FROM embeddings WHERE model = ? AND content_hash = ? AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ?)");
-    for (const hash of unique)
-        for (const model of models)
-            purge.run(model, hash, hash);
-};
 /** Each attachment's extracted text, in pieces for search by meaning, as an email's own text is. */
 export const chunkAttachments = (database, accountId, emailId, now) => {
     const email = database
@@ -125,7 +109,7 @@ export const drainEmailIndex = (database, stemmerFor) => {
                 .prepare("SELECT id FROM email_threads")
                 .all()
                 .map(({ id }) => Number(id)));
-            purgeUnused(database, single);
+            dropUnusedVectors(database, single);
             database
                 .prepare("INSERT OR REPLACE INTO store_settings (key, value, at) VALUES (?, ?, ?)")
                 .run(MAIL_CUT, CUT, Date.now());
@@ -146,7 +130,7 @@ export const drainEmailIndex = (database, stemmerFor) => {
             for (const row of unchunked) {
                 const before = dropAttachmentChunks(database, Number(row.id));
                 chunkAttachments(database, Number(row.account_id), Number(row.id), Date.now());
-                purgeUnused(database, before);
+                dropUnusedVectors(database, before);
             }
         });
     const emails = drainCorpus(database, CORPORA.email, stemmerFor);

package/dist/store/sqlite/emails.js

@@ -1,7 +1,7 @@
 import { formatLocator } from "../../domain/locator.js";
 import { formatReference } from "../../domain/references.js";
 import { normalize } from "../normalize.js";
-import { chunkAttachments, drainEmailIndex, dropAttachmentChunks, MAIL_PIECES, purgeUnused, queueThreads, threadPieces, titleOf, } from "./email-index.js";
+import { chunkAttachments, drainEmailIndex, dropAttachmentChunks, MAIL_PIECES, queueThreads, threadPieces, titleOf, } from "./email-index.js";
 import { recountEmailThread as recount, setEmailFlags } from "./email-state.js";
 import { bool, flag, fromJson, int, page, str, toJson } from "./events.js";
 import { ensurePerson } from "./identities.js";
@@ -9,6 +9,7 @@ import { mailRulesOver } from "./mail-rules.js";
 import { attachmentOf, saveAttachment, wordsQuery } from "./meetings.js";
 import { stemmerCache } from "./stems.js";
 import { write } from "./transaction.js";
+import { dropUnusedVectors } from "./vector-index.js";
 import { dot } from "./vectors.js";
 /** A thread as a conversation: the summary `search conversations` and `related` answer for an email. */
 export const threadConversation = (place, builtAt) => ({
@@ -234,7 +235,7 @@ const saveEmail = (database, orm, accountId, threadId, mail, now) => {
         for (const a of mail.attachments)
             saveAttachment(database, "email", emailId, { ...a, normalizedText: a.text === null ? null : normalize(a.text) }, now);
         chunkAttachments(database, accountId, emailId, now);
-        purgeUnused(database, before);
+        dropUnusedVectors(database, before);
     }
     return found ? Number(found.email_thread_id) : null;
 };
@@ -547,7 +548,7 @@ export const mailStoreOver = ({ database, orm }, stemmerFor = stemmerCache()) =>
             const threads = externalIds.flatMap((id) => mark.all(now, now, accountId, id).map((r) => {
                 const pieces = dropAttachmentChunks(database, Number(r.id));
                 forget.run(Number(r.id));
-                purgeUnused(database, pieces);
+                dropUnusedVectors(database, pieces);
                 return Number(r.email_thread_id);
             }));
             for (const id of new Set(threads))

package/dist/store/sqlite/image-chunks.d.ts

@@ -34,7 +34,7 @@ export interface ImageChunk {
  * Makes these the chunks of their images, and drops the chunks of images that are gone or whose message was
  * deleted, with the vectors nothing else uses any more.
  */
-export declare const saveImageChunks: ({ orm, now }: StoreContext, accountPk: number, chatKey: number, chunks: ImageChunk[], keep: number[]) => void;
+export declare const saveImageChunks: ({ orm, now, database }: StoreContext, accountPk: number, chatKey: number, chunks: ImageChunk[], keep: number[]) => void;
 /** The hashes among these that already have a vector of `model`. */
 export declare const embeddedHashes: ({ orm }: StoreContext, model: string, hashes: string[]) => Set<string>;
 export interface NearestImage {
@@ -75,7 +75,7 @@ export declare const picturesToEmbed: ({ orm }: StoreContext, accountPk: number,
     limit: number;
 }) => PictureInput[];
 /** Makes each hash the picture chunk of its image; the vectors are saved apart, by `saveVectors`. */
-export declare const savePictureChunks: ({ orm, now }: StoreContext, accountPk: number, pictures: {
+export declare const savePictureChunks: ({ orm, now, database }: StoreContext, accountPk: number, pictures: {
     attachmentPk: number;
     hash: string;
     bytes: number;

package/dist/store/sqlite/image-chunks.js

@@ -1,5 +1,5 @@
 import { sql } from "./drizzle/core.js";
-import { searchIndex } from "./vector-index.js";
+import { dropUnusedVectors, searchIndex } from "./vector-index.js";
 import { dot } from "./vectors.js";
 const IS_IMAGE = sql `(a.kind = 'photo' OR a.mime LIKE 'image/%')`;
 const neighbours = ({ orm }, chatKey, messagePk, sentAt, { messages, withinMs }, side) => orm.all(sql `SELECT sender_name AS sender, text, sent_at AS sentAt FROM messages
@@ -27,7 +27,7 @@ export const imagesOfChat = (context, chatKey, window) => context.orm
  * Makes these the chunks of their images, and drops the chunks of images that are gone or whose message was
  * deleted, with the vectors nothing else uses any more.
  */
-export const saveImageChunks = ({ orm, now }, accountPk, chatKey, chunks, keep) => {
+export const saveImageChunks = ({ orm, now, database }, accountPk, chatKey, chunks, keep) => {
     const at = now();
     const scope = orm.get(sql `SELECT scope FROM accounts WHERE id = ${accountPk}`)?.scope ?? null;
     const old = new Set();
@@ -51,9 +51,7 @@ export const saveImageChunks = ({ orm, now }, accountPk, chatKey, chunks, keep)
           account_id, occurred_at, created_at, updated_at)
         VALUES ('image', ${attachmentPk}, 0, 0, ${length}, ${hash}, ${scope}, ${accountPk}, ${occurredAt}, ${at}, ${at})`);
     }
-    for (const hash of old)
-        orm.run(sql `DELETE FROM embeddings WHERE content_hash = ${hash}
-      AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ${hash})`);
+    dropUnusedVectors(database, old);
 };
 /** The hashes among these that already have a vector of `model`. */
 export const embeddedHashes = ({ orm }, model, hashes) => new Set(hashes.length === 0
@@ -135,24 +133,23 @@ export const picturesToEmbed = ({ orm }, accountPk, { chatKey, after, limit }) =
           AND k.position = ${PICTURE_POSITION})
       ORDER BY a.id LIMIT ${limit}`);
 /** Makes each hash the picture chunk of its image; the vectors are saved apart, by `saveVectors`. */
-export const savePictureChunks = ({ orm, now }, accountPk, pictures) => {
+export const savePictureChunks = ({ orm, now, database }, accountPk, pictures) => {
     const at = now();
     const scope = orm.get(sql `SELECT scope FROM accounts WHERE id = ${accountPk}`)?.scope ?? null;
     // A deleted message keeps no trace of its picture, as it keeps none of its text.
     const gone = sql `chunkable_type = 'image' AND position = ${PICTURE_POSITION} AND account_id = ${accountPk}
     AND NOT EXISTS (SELECT 1 FROM attachments a JOIN messages m ON a.attachable_type = 'message'
       AND m.id = a.attachable_id WHERE a.id = chunkable_id AND m.deleted_at IS NULL)`;
-    const dropped = orm.all(sql `SELECT content_hash AS hash FROM chunks WHERE ${gone}`);
-    orm.run(sql `DELETE FROM chunks WHERE ${gone}`);
-    for (const { hash } of dropped)
-        orm.run(sql `DELETE FROM embeddings WHERE content_hash = ${hash}
-      AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ${hash})`);
+    const released = orm.all(sql `DELETE FROM chunks WHERE ${gone} RETURNING content_hash AS hash`);
     for (const { attachmentPk, hash, bytes, occurredAt } of pictures) {
-        orm.run(sql `DELETE FROM chunks WHERE chunkable_type = 'image' AND chunkable_id = ${attachmentPk}
-        AND position = ${PICTURE_POSITION}`);
+        const replaced = orm.get(sql `DELETE FROM chunks WHERE chunkable_type = 'image' AND chunkable_id = ${attachmentPk}
+        AND position = ${PICTURE_POSITION} RETURNING content_hash AS hash`);
+        if (replaced)
+            released.push(replaced);
         orm.run(sql `INSERT INTO chunks (chunkable_type, chunkable_id, position, start_offset, end_offset, content_hash, scope,
           account_id, occurred_at, created_at, updated_at)
         VALUES ('image', ${attachmentPk}, ${PICTURE_POSITION}, 0, ${bytes}, ${hash}, ${scope}, ${accountPk},
           ${occurredAt}, ${at}, ${at})`);
     }
+    dropUnusedVectors(database, released.map(({ hash }) => hash));
 };

package/dist/store/sqlite/meeting-vectors.js

@@ -3,6 +3,7 @@ import { chunkHash, cutChunks } from "../../conversations/chunks.js";
 import { positiveReadInteger } from "./meeting-reads.js";
 import { meetingSelectionSql } from "./meeting-selection.js";
 import { read, write } from "./transaction.js";
+import { dropUnusedVectors } from "./vector-index.js";
 import { dot, saveVectors } from "./vectors.js";
 /** Vectors read per query while ranking, so a store of any size stays within a few megabytes at a time. */
 const RANK_BATCH = 512;
@@ -129,11 +130,13 @@ export const meetingVectorsOver = (context) => ({
                 text: load(Number(row.id)),
             }));
             let count = 0;
+            const released = [];
             for (const { row, text } of prepared) {
                 input.signal?.throwIfAborted();
-                context.database
-                    .prepare("DELETE FROM chunks WHERE chunkable_type='meeting_transcript' AND chunkable_id=?")
-                    .run(Number(row.id));
+                for (const { hash } of context.database
+                    .prepare("DELETE FROM chunks WHERE chunkable_type='meeting_transcript' AND chunkable_id=? RETURNING content_hash AS hash")
+                    .all(Number(row.id)))
+                    released.push(String(hash));
                 const chunks = cutChunks([{ id: String(row.id), sender: null, text: text.text }], undefined, () => input.signal?.throwIfAborted());
                 for (let index = 0; index < chunks.length; index++) {
                     const chunk = chunks[index];
@@ -145,12 +148,11 @@ export const meetingVectorsOver = (context) => ({
                     count++;
                 }
             }
-            context.database
-                .prepare("DELETE FROM embeddings WHERE EXISTS (SELECT 1 FROM chunks k WHERE k.content_hash=embeddings.content_hash AND k.chunkable_type='meeting_transcript' AND k.account_id=? AND NOT EXISTS (SELECT 1 FROM meeting_transcripts t JOIN meetings m ON m.id=t.meeting_id WHERE t.id=k.chunkable_id AND t.deleted_at IS NULL AND t.superseded_at IS NULL AND m.deleted_at IS NULL AND m.account_id=?)) AND NOT EXISTS (SELECT 1 FROM chunks k WHERE k.content_hash=embeddings.content_hash AND (k.chunkable_type <> 'meeting_transcript' OR k.account_id IS NOT ? OR EXISTS (SELECT 1 FROM meeting_transcripts t JOIN meetings m ON m.id=t.meeting_id WHERE t.id=k.chunkable_id AND t.deleted_at IS NULL AND t.superseded_at IS NULL AND m.deleted_at IS NULL AND m.account_id=?)))")
-                .run(input.accountId, input.accountId, input.accountId, input.accountId);
-            context.database
-                .prepare("DELETE FROM chunks WHERE chunkable_type='meeting_transcript' AND account_id=? AND NOT EXISTS (SELECT 1 FROM meeting_transcripts t JOIN meetings m ON m.id=t.meeting_id WHERE t.id=chunks.chunkable_id AND t.deleted_at IS NULL AND t.superseded_at IS NULL AND m.deleted_at IS NULL AND m.account_id=?)")
-                .run(input.accountId, input.accountId);
+            for (const { hash } of context.database
+                .prepare("DELETE FROM chunks WHERE chunkable_type='meeting_transcript' AND account_id=? AND NOT EXISTS (SELECT 1 FROM meeting_transcripts t JOIN meetings m ON m.id=t.meeting_id WHERE t.id=chunks.chunkable_id AND t.deleted_at IS NULL AND t.superseded_at IS NULL AND m.deleted_at IS NULL AND m.account_id=?) RETURNING content_hash AS hash")
+                .all(input.accountId, input.accountId))
+                released.push(String(hash));
+            dropUnusedVectors(context.database, released);
             input.signal?.throwIfAborted();
             const hasMore = found.length > limit;
             return {

package/dist/store/sqlite/message-chunks.js

@@ -1,5 +1,5 @@
 import { sql } from "./drizzle/core.js";
-import { searchIndex } from "./vector-index.js";
+import { dropUnusedVectors, searchIndex } from "./vector-index.js";
 import { dot } from "./vectors.js";
 /** Live messages with text of the account, by pk after `after`; the caller compares hashes. */
 export const messagesToEmbed = ({ orm }, accountPk, { chatKey, model, after, limit }) => orm
@@ -11,11 +11,6 @@ export const messagesToEmbed = ({ orm }, accountPk, { chatKey, model, after, lim
           ${chatKey === undefined ? sql `` : sql `AND m.chat_id = ${chatKey}`}
         ORDER BY m.id LIMIT ${limit}`)
     .map((row) => ({ ...row, embedded: row.embedded === 1 }));
-const dropUnshared = ({ orm }, hashes) => {
-    for (const hash of hashes)
-        orm.run(sql `DELETE FROM embeddings WHERE content_hash = ${hash}
-      AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ${hash})`);
-};
 /** Makes each hash its message's chunk; the vectors are saved apart, by `saveVectors`. */
 export const saveMessageChunks = (context, accountPk, chunks) => {
     const { orm, now } = context;
@@ -34,7 +29,7 @@ export const saveMessageChunks = (context, accountPk, chunks) => {
           account_id, occurred_at, created_at, updated_at)
         VALUES ('message', ${messagePk}, 0, 0, ${length}, ${hash}, ${scope}, ${accountPk}, ${occurredAt}, ${at}, ${at})`);
     }
-    dropUnshared(context, old);
+    dropUnusedVectors(context.database, old);
 };
 /**
  * Drops the chunks of the account's messages that are gone, deleted or emptied, with the vectors nothing else
@@ -45,7 +40,7 @@ export const dropGoneMessageChunks = (context, accountPk) => {
     WHERE m.id = chunkable_id AND m.deleted_at IS NULL AND trim(m.text) <> '')`;
     const dropped = context.orm.all(sql `SELECT content_hash AS hash FROM chunks WHERE ${gone}`);
     context.orm.run(sql `DELETE FROM chunks WHERE ${gone}`);
-    dropUnshared(context, dropped.map(({ hash }) => hash));
+    dropUnusedVectors(context.database, dropped.map(({ hash }) => hash));
     return dropped.length;
 };
 /** Whether any message of the account has a vector of `model`. */