@wirecat/cli-core 0.20.0 → 0.263.0

package/dist/store/store.js

@@ -0,0 +1,949 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { CliError } from "@wirecat/cli-base";
+import { ACCOUNT_GROUPS } from "../search/account-list.js";
+import { bindAccount } from "./account-store.js";
+import { migrate } from "./migrations.js";
+import { removeOldStore } from "./old-store.js";
+import { storeCapable } from "./open.js";
+import { storePath } from "./path.js";
+import { accountGroupsOf, profilesOf, saveProfile } from "./sqlite/account-groups.js";
+import * as accounts from "./sqlite/accounts.js";
+import { adminStatisticsQuery } from "./sqlite/admin-statistics.js";
+import { agentActionsStoreOver } from "./sqlite/agent-actions.js";
+import * as attachmentTexts from "./sqlite/attachment-texts.js";
+import * as attachmentRows from "./sqlite/attachments.js";
+import { backfillNormalized, pendingNormalization } from "./sqlite/backfill.js";
+import * as batches from "./sqlite/batches.js";
+import { botUpdateStoreOver } from "./sqlite/bot-updates.js";
+import { catalogStoreOver } from "./sqlite/catalog.js";
+import * as metadataQueries from "./sqlite/chats.js";
+import * as chatQueries from "./sqlite/chats.js";
+import * as completeness from "./sqlite/completeness.js";
+import { conversationEligibility } from "./sqlite/conversation-eligibility.js";
+import * as conversationQueries from "./sqlite/conversations.js";
+import { applyCounterObservations, counterStates, counterTargets } from "./sqlite/counters.js";
+import { documentLinkMessages } from "./sqlite/document-links.js";
+import { drainEmailIndex, mailStoreOver, threadConversation } from "./sqlite/emails.js";
+import { factsStoreOver } from "./sqlite/facts.js";
+import * as identities from "./sqlite/identities.js";
+import * as imageChunks from "./sqlite/image-chunks.js";
+import { drainInvolvementQueue } from "./sqlite/involvement-queue.js";
+import { involvementStoreOver } from "./sqlite/involvements.js";
+import { knowledgeStoreOver } from "./sqlite/knowledge.js";
+import { findRegex } from "./sqlite/legacy-regex.js";
+import { logPruner } from "./sqlite/log-pruning.js";
+import * as lucene from "./sqlite/lucene.js";
+import { meetingVectorsOver } from "./sqlite/meeting-vectors.js";
+import { meetingStoreOver } from "./sqlite/meetings.js";
+import { memoriesStoreOver } from "./sqlite/memories.js";
+import * as messageWrites from "./sqlite/messages.js";
+import { noteSearchOver } from "./sqlite/note-search.js";
+import { notesStoreOver } from "./sqlite/notes.js";
+import { openSqlite } from "./sqlite/open.js";
+import * as personLinks from "./sqlite/person-links.js";
+import { personSeenInAccounts } from "./sqlite/person-scope.js";
+import * as privatePeople from "./sqlite/private-people.js";
+import { proposedActionsStoreOver } from "./sqlite/proposed-actions.js";
+import * as ranges from "./sqlite/ranges.js";
+import { rankingEvidence } from "./sqlite/ranking-evidence.js";
+import { rankQuery } from "./sqlite/rankings.js";
+import * as reads from "./sqlite/reads.js";
+import { retentionQuery } from "./sqlite/retention.js";
+import * as roster from "./sqlite/roster.js";
+import * as search from "./sqlite/search.js";
+import * as searchIndex from "./sqlite/search-index.js";
+import * as searchRecords from "./sqlite/searches.js";
+import * as stems from "./sqlite/stems.js";
+import * as sync from "./sqlite/sync.js";
+import * as tagQueries from "./sqlite/tags.js";
+import { linkInbox, taskStoreOver } from "./sqlite/tasks.js";
+import * as transcripts from "./sqlite/transcripts.js";
+import { toMs } from "./sqlite/values.js";
+import * as vectors from "./sqlite/vectors.js";
+import * as words from "./sqlite/words.js";
+/** About 40 ms of the first open (129k rows/s measured at 1M, 2026-09-30); Tab opens the store too. */
+export const BACKFILL_ON_OPEN = 5_000;
+export const openStore = async ({ path, env, now = Date.now, command } = {}) => {
+    await storeCapable();
+    const file = path ?? storePath(env);
+    mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
+    if (path === undefined && (env ?? process.env).MESSAGING_STORE === undefined) {
+        const removed = await removeOldStore(dirname(file)).catch(() => undefined);
+        if (removed)
+            process.stderr.write(`Deleted the old message store ${removed}; everything now lives in ${file}.\n`);
+    }
+    // Created before SQLite opens it: SQLite gives -wal and -shm the mode of the database file.
+    writeFileSync(file, "", { flag: "a", mode: 0o600 });
+    const { database, orm } = await openSqlite(file);
+    try {
+        migrate(database, { now, command });
+        // A small file is filled on the spot; a larger one waits for `db migrate`, since nothing reads the copy yet.
+        const pending = pendingNormalization(database);
+        if (pending > 0 && pending <= BACKFILL_ON_OPEN)
+            backfillNormalized(database);
+        stems.followDefaultStemmers(database, now);
+        const stemming = stems.stemsState(database);
+        if (stemming?.cause === "building" &&
+            stemming.watermark - stemming.filledThrough + stemming.pending <= BACKFILL_ON_OPEN) {
+            stems.fillStems(database, { now });
+        }
+        logPruner(database, now)();
+    }
+    catch (error) {
+        database.close();
+        throw error;
+    }
+    return storeOver({ database, orm, now });
+};
+const storeOver = (context) => {
+    const { database } = context;
+    const stemmerFor = stems.stemmerCache();
+    const pruneLogsWhenDue = logPruner(database, context.now);
+    const inTransaction = (body) => {
+        database.exec("BEGIN IMMEDIATE");
+        try {
+            body();
+            stems.drainStems(database, stemmerFor);
+            drainInvolvementQueue(database);
+            database.exec("COMMIT");
+        }
+        catch (error) {
+            database.exec("ROLLBACK");
+            throw error;
+        }
+    };
+    const mailIndexed = (execution) => {
+        if (execution.corpus === "mail")
+            drainEmailIndex(database, stemmerFor);
+        return execution;
+    };
+    const accountPk = (key, name = null) => accounts.accountPk(context, key, name);
+    const findAccountPk = (key) => accounts.findAccountPk(context, key);
+    const findChatPk = (accountKey, chatId) => chatQueries.findChatPk(context, accountKey, chatId);
+    const chatKeyOf = (key, chatId) => {
+        const accountKey = findAccountPk(key);
+        return accountKey === undefined ? undefined : findChatPk(accountKey, chatId);
+    };
+    const chatPkFor = (accountKey, chatId) => chatQueries.chatPkFor(context, accountKey, chatId);
+    const identityPk = (accountKey, provider, nativeId, name, facts) => identities.identityPk(context, accountKey, provider, nativeId, name, facts);
+    const upsertMessage = (key, accountKey, chatKey, message, via, seenAt) => messageWrites.upsertMessage(context, key, accountKey, chatKey, message, via, seenAt);
+    const writeMembers = (key, accountKey, chatId, memberIds) => chatQueries.writeMembers(context, key, accountKey, chatId, memberIds);
+    const writeState = (accountKey, name, value) => sync.writeState(context, accountKey, name, value);
+    let botUpdates;
+    let involvements;
+    let tasks;
+    let knowledge;
+    let notes;
+    let catalog;
+    let facts;
+    let memories;
+    let proposedActions;
+    let agentActions;
+    let meetings;
+    let mail;
+    const mailOf = () => {
+        mail ??= mailStoreOver(context, stemmerFor);
+        return mail;
+    };
+    const store = {
+        meetingVectors: meetingVectorsOver(context),
+        saveAccount: async (key, { name }) => {
+            const pk = accountPk(key, name);
+            linkInbox(database, `${key.provider}:${key.account}`, pk);
+            return pk;
+        },
+        saveChats: async (key, list) => inTransaction(() => {
+            const accountKey = accountPk(key);
+            for (const chat of list)
+                chatQueries.upsertChat(context, accountKey, chat);
+        }),
+        saveMessages: async (key, chatId, messages, { via, seenAt }) => inTransaction(() => {
+            const accountKey = accountPk(key);
+            const chatKey = chatPkFor(accountKey, chatId);
+            for (const message of messages) {
+                if (message.scheduledFor === undefined)
+                    upsertMessage(key, accountKey, chatKey, message, via, seenAt);
+            }
+        }),
+        saveMembers: async (key, chatId, memberIds) => inTransaction(() => writeMembers(key, accountPk(key), chatId, memberIds)),
+        saveRoster: async (key, chatId, read) => {
+            let change = { joined: [], gone: [], changed: [] };
+            inTransaction(() => {
+                const accountKey = accountPk(key);
+                change = roster.saveRoster(context, accountKey, key.provider, chatPkFor(accountKey, chatId), read);
+            });
+            return change;
+        },
+        memberStays: async (key, chatId, { since } = {}) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined
+                ? []
+                : roster.memberStaysOf(context, chatKey, since === undefined ? undefined : Date.parse(since));
+        },
+        memberCounts: async (key, chatId, { since } = {}) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? [] : roster.memberCountsOf(context, chatKey, since);
+        },
+        profileRevisions: async (key, chatId, { since } = {}) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined
+                ? []
+                : roster.profileRevisionsOf(context, chatKey, since === undefined ? undefined : Date.parse(since));
+        },
+        trackMembers: async (key, chatId, tracked) => inTransaction(() => roster.setTracked(context, chatPkFor(accountPk(key), chatId), tracked)),
+        trackedChats: async (key) => {
+            const accountKey = findAccountPk(key);
+            return accountKey === undefined ? [] : roster.trackedChats(context, accountKey);
+        },
+        members: async (key, chatId) => {
+            const accountKey = findAccountPk(key);
+            const chatKey = accountKey === undefined ? undefined : findChatPk(accountKey, chatId);
+            return chatKey === undefined ? [] : chatQueries.members(context, chatKey);
+        },
+        personOf: async (identity) => personLinks.personOf(context, identity),
+        personSeenInAccounts: async (uid, accountIds) => personSeenInAccounts(context, uid, accountIds),
+        personByUid: async (uid) => personLinks.personByUid(context, uid),
+        linkIdentities: async (person, other, options) => {
+            let linked;
+            inTransaction(() => {
+                linked = personLinks.linkIdentities(context, person, other, options);
+            });
+            return linked;
+        },
+        unlinkIdentity: async (identity, options) => {
+            let alone;
+            inTransaction(() => {
+                alone = personLinks.unlinkIdentity(context, identity, options);
+            });
+            return alone;
+        },
+        mentioning: async (key, person, limit) => {
+            const accountKey = findAccountPk(key);
+            return accountKey === undefined
+                ? { items: [], hasMore: false }
+                : personLinks.mentioning(context, accountKey, person, limit);
+        },
+        chatsWith: async (key, memberId) => {
+            const accountKey = findAccountPk(key);
+            return accountKey === undefined ? [] : chatQueries.chatsWith(context, accountKey, key, memberId);
+        },
+        syncState: async (key, name) => {
+            const accountKey = findAccountPk(key);
+            return accountKey === undefined ? undefined : sync.syncStateOf(context, accountKey, name);
+        },
+        setSyncState: async (key, name, value) => inTransaction(() => writeState(accountPk(key), name, value)),
+        applyDelta: async (key, { chats = [], people = [], members = new Map(), state = {} }) => inTransaction(() => {
+            const accountKey = accountPk(key);
+            for (const chat of chats)
+                chatQueries.upsertChat(context, accountKey, chat);
+            for (const person of people)
+                identityPk(accountKey, key.provider, person.id, person.name, person);
+            for (const [chatId, ids] of members)
+                writeMembers(key, accountKey, chatId, ids);
+            for (const [name, value] of Object.entries(state))
+                writeState(accountKey, name, value);
+        }),
+        clearSyncState: async (key, name) => {
+            const accountKey = findAccountPk(key);
+            if (accountKey !== undefined)
+                sync.clearState(context, accountKey, name);
+        },
+        claim: async (key, chatId, anchor, holder, forMs) => {
+            let taken = false;
+            inTransaction(() => {
+                taken = sync.claim(context, chatPkFor(accountPk(key), chatId), anchor, holder, forMs);
+            });
+            return taken;
+        },
+        release: async (key, chatId, anchor, holder) => {
+            const accountKey = findAccountPk(key);
+            const chatKey = accountKey === undefined ? undefined : findChatPk(accountKey, chatId);
+            if (chatKey !== undefined)
+                sync.release(context, chatKey, anchor, holder);
+        },
+        transcript: async (key, chatId, messageId) => {
+            const accountKey = findAccountPk(key);
+            const chatKey = accountKey === undefined ? undefined : findChatPk(accountKey, chatId);
+            return chatKey === undefined ? undefined : transcripts.transcript(context, chatKey, messageId);
+        },
+        keepTranscript: async (key, chatId, messageId, text, source) => {
+            if (text.trim() === "")
+                return;
+            inTransaction(() => transcripts.keepTranscript(context, chatPkFor(accountPk(key), chatId), messageId, text, source));
+        },
+        fileAttachments: async (key, { chatId, ...page }) => {
+            const accountKey = findAccountPk(key);
+            if (accountKey === undefined)
+                return [];
+            if (chatId === undefined)
+                return attachmentTexts.fileAttachments(context, accountKey, page);
+            const chatKey = findChatPk(accountKey, chatId);
+            return chatKey === undefined ? [] : attachmentTexts.fileAttachments(context, accountKey, { ...page, chatKey });
+        },
+        attachments: async (key, { chatId, ...filter }) => {
+            const accountKey = findAccountPk(key);
+            if (accountKey === undefined)
+                return [];
+            if (chatId === undefined)
+                return attachmentTexts.attachmentViews(context, accountKey, filter);
+            const chatKey = findChatPk(accountKey, chatId);
+            return chatKey === undefined ? [] : attachmentTexts.attachmentViews(context, accountKey, { ...filter, chatKey });
+        },
+        localPathOf: async (attachmentPk) => attachmentTexts.localPathOf(context, attachmentPk),
+        keepAttachmentText: async (attachmentPk, entry) => {
+            let kept = false;
+            inTransaction(() => {
+                kept = attachmentTexts.keepText(context, attachmentPk, entry);
+            });
+            return kept;
+        },
+        keepDownloads: async (key, chatId, messageId, files) => {
+            const chatKey = chatKeyOf(key, chatId);
+            if (chatKey === undefined || files.length === 0)
+                return 0;
+            let kept = 0;
+            inTransaction(() => {
+                kept = attachmentRows.keepDownloads(context, chatKey, messageId, files);
+            });
+            return kept;
+        },
+        linkInputs: async (key, chatId, page) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? { items: [], next: null } : conversationQueries.linkInputs(context, chatKey, page);
+        },
+        senderHandles: async (key, chatId) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? new Map() : conversationQueries.senderHandles(context, chatKey);
+        },
+        replaceConversations: async (key, chatId, build) => {
+            const chatKey = chatKeyOf(key, chatId);
+            if (chatKey === undefined)
+                throw new CliError("not_found", `chat ${chatId} is not in the local copy`);
+            await conversationQueries.replaceConversations(context, chatKey, build);
+        },
+        conversations: async (key, chatId, { limit, after, before }) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined
+                ? { items: [], hasMore: false }
+                : conversationQueries.conversationPage(context, chatKey, {
+                    limit,
+                    ...(after === undefined ? {} : { after: toMs(after) }),
+                    ...(before === undefined ? {} : { before: toMs(before) }),
+                });
+        },
+        conversation: async (key, id) => {
+            const accountKey = findAccountPk(key);
+            const pk = Number(id);
+            return accountKey === undefined || !Number.isSafeInteger(pk)
+                ? undefined
+                : conversationQueries.conversation(context, accountKey, pk);
+        },
+        conversationOf: async (key, chatId, messageId) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? undefined : conversationQueries.conversationOf(context, chatKey, messageId);
+        },
+        conversationsOfMessages: async (key, refs) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return refs.map(() => undefined);
+            const pks = refs.map(({ chatId, messageId }) => {
+                const chatKey = findChatPk(accountPk, chatId);
+                const id = chatKey === undefined ? undefined : conversationQueries.conversationOf(context, chatKey, messageId);
+                return id === undefined ? undefined : Number(id);
+            });
+            const found = conversationQueries.summariesOf(context, accountPk, pks.filter((pk) => pk !== undefined));
+            return pks.map((pk) => (pk === undefined ? undefined : found.get(pk)));
+        },
+        links: async (key, chatId, messageId, options) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? [] : conversationQueries.linksOf(context, chatKey, messageId, options?.limit);
+        },
+        replies: async (key, chatId, messageId, limit) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined
+                ? { items: [], hasMore: false }
+                : conversationQueries.repliesTo(context, chatKey, messageId, limit);
+        },
+        agentAnswers: async (key, chatId) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? new Map() : conversationQueries.agentAnswers(context, chatKey);
+        },
+        saveAnswers: async (key, batch, answer) => {
+            const chatKey = batches.chatOfBatch(batch);
+            const accountKey = findAccountPk(key);
+            const chat = chatKey === undefined || accountKey === undefined ? undefined : chatQueries.chatOf(context, accountKey, chatKey);
+            if (chatKey === undefined || chat === undefined) {
+                throw new CliError("validation_error", `the answer was not stored: ${batch} is not a batch of this account`);
+            }
+            let stored = 0;
+            inTransaction(() => {
+                stored = batches.saveAnswers(context, batch, answer);
+            });
+            return { chat, stored };
+        },
+        clearAnswers: async (key, chatId, model) => {
+            const chatKey = chatKeyOf(key, chatId);
+            if (chatKey === undefined)
+                return 0;
+            let cleared = 0;
+            inTransaction(() => {
+                cleared = batches.clearAnswers(context, chatKey, model);
+            });
+            return cleared;
+        },
+        batchStatus: async (key, chatId) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? { messages: 0, characters: 0 } : batches.batchStatus(context, chatKey);
+        },
+        nextBatch: async (key, chatId, { size }) => {
+            const chatKey = chatKeyOf(key, chatId);
+            if (chatKey === undefined)
+                return undefined;
+            const found = batches.nextBatch(context, chatKey, chatId, size);
+            if (!found)
+                return undefined;
+            const answered = found.messages.filter(({ answer }) => answer);
+            const left = batches.batchStatus(context, chatKey);
+            return {
+                ...found,
+                remaining: {
+                    messages: left.messages - answered.length,
+                    characters: left.characters - answered.reduce((sum, { text }) => sum + [...text].length, 0),
+                },
+            };
+        },
+        chunksToEmbed: async (key, chatId, model, options) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? [] : vectors.chunksToEmbed(context, chatKey, model, options);
+        },
+        saveVectors: async (model, dims, rows) => {
+            inTransaction(() => vectors.saveVectors(context, model, dims, rows));
+        },
+        imagesToDescribe: async (key, chatId, window) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? [] : imageChunks.imagesOfChat(context, chatKey, window);
+        },
+        saveImageChunks: async (key, chatId, chunks, keep) => {
+            const accountPk = findAccountPk(key);
+            const chatKey = chatKeyOf(key, chatId);
+            if (accountPk === undefined || chatKey === undefined)
+                return;
+            inTransaction(() => imageChunks.saveImageChunks(context, accountPk, chatKey, chunks, keep));
+        },
+        embeddedHashes: async (model, hashes) => imageChunks.embeddedHashes(context, model, hashes),
+        nearestImages: async (key, { chatId, model, since, limit, query }) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return [];
+            const chatKey = chatId === undefined ? undefined : chatKeyOf(key, chatId);
+            if (chatId !== undefined && chatKey === undefined)
+                return [];
+            const nearest = imageChunks.nearestImages(context, accountPk, {
+                ...(chatKey === undefined ? {} : { chatKey }),
+                ...(since === undefined ? {} : { since: Date.parse(since) }),
+                model,
+                limit,
+                query,
+            });
+            const hits = [];
+            for (const { chatId: chat, messageId, score } of nearest) {
+                const chatPk = findChatPk(accountPk, chat);
+                const conversation = chatPk === undefined ? undefined : conversationQueries.conversationOf(context, chatPk, messageId);
+                const summary = conversation === undefined
+                    ? undefined
+                    : conversationQueries.summariesOf(context, accountPk, [Number(conversation)]).get(Number(conversation));
+                if (summary)
+                    hits.push({
+                        summary,
+                        chunk: { firstMessageId: messageId, lastMessageId: messageId },
+                        score,
+                        stale: false,
+                        matchedIn: "context",
+                    });
+            }
+            return hits;
+        },
+        vectorStatus: async (key, chatId, model) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? { chunks: 0, embedded: 0 } : vectors.vectorStatus(context, chatKey, model);
+        },
+        clearVectors: async (key, chatId, model) => {
+            const chatKey = chatKeyOf(key, chatId);
+            if (chatKey === undefined)
+                return 0;
+            let cleared = 0;
+            inTransaction(() => {
+                cleared = vectors.clearVectors(context, chatKey, model);
+            });
+            return cleared;
+        },
+        nearestConversations: async (key, { chatId, model, since, limit, query, exclude, conversations, scope, projectId, personId, before }) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return [];
+            const chatKey = chatId === undefined ? undefined : chatKeyOf(key, chatId);
+            const mailToo = key.provider === "email" &&
+                conversations === undefined &&
+                scope === undefined &&
+                projectId === undefined &&
+                personId === undefined &&
+                before === undefined;
+            const mailHits = async () => mailToo
+                ? (await mailOf().nearest(model, query, {
+                    accountId: accountPk,
+                    limit,
+                    ...(chatId === undefined ? {} : { threadExternalId: chatId }),
+                    ...(since === undefined ? {} : { since: Date.parse(since) }),
+                })).map((hit) => ({
+                    summary: threadConversation(hit, context.now()),
+                    chunk: { firstMessageId: hit.externalId, lastMessageId: hit.externalId },
+                    score: hit.score,
+                    stale: false,
+                }))
+                : [];
+            if (chatId !== undefined && chatKey === undefined)
+                return mailHits();
+            const nearest = vectors.nearestChunks(context, accountPk, {
+                ...(chatKey === undefined ? {} : { chatKey }),
+                ...(since === undefined ? {} : { since: Date.parse(since) }),
+                ...(exclude === undefined ? {} : { exclude: Number(exclude) }),
+                model,
+                limit,
+                query,
+                ...(conversations === undefined ? {} : { conversations }),
+                ...(scope === undefined ? {} : { scope }),
+                ...(projectId === undefined ? {} : { projectId: Number(projectId) }),
+                ...(personId === undefined ? {} : { personId: Number(personId) }),
+                ...(before === undefined ? {} : { before: Date.parse(before) }),
+            });
+            const found = conversationQueries.summariesOf(context, accountPk, nearest.map(({ conversationPk }) => conversationPk));
+            const ids = vectors.messageIds(context, nearest.flatMap(({ firstMessagePk, lastMessagePk }) => [firstMessagePk, lastMessagePk]));
+            const messageHits = nearest.flatMap((chunk) => {
+                const summary = found.get(chunk.conversationPk);
+                const first = ids.get(chunk.firstMessagePk);
+                const last = ids.get(chunk.lastMessagePk);
+                const freshness = vectors.chunkFreshness(context, chunk);
+                return summary && first && last && freshness !== "deleted"
+                    ? [
+                        {
+                            summary,
+                            chunk: { firstMessageId: first, lastMessageId: last },
+                            score: chunk.score,
+                            stale: freshness === "stale",
+                        },
+                    ]
+                    : [];
+            });
+            if (!mailToo)
+                return messageHits;
+            return [...messageHits, ...(await mailHits())].sort((a, b) => b.score - a.score).slice(0, limit);
+        },
+        conversationVectors: async (key, id, model) => {
+            const accountPk = findAccountPk(key);
+            const pk = Number(id);
+            if (accountPk === undefined || !conversationQueries.summariesOf(context, accountPk, [pk]).has(pk)) {
+                return { chunks: 0, vectors: [] };
+            }
+            return vectors.conversationVectors(context, pk, model);
+        },
+        readiness: async (key, { chatId, model }) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return [];
+            const chats = chatId === undefined
+                ? vectors.builtChats(context, accountPk)
+                : [{ chatKey: chatKeyOf(key, chatId), id: chatId }].flatMap(({ chatKey, id }) => chatKey === undefined ? [] : [{ chatKey, id }]);
+            return chats.map(({ chatKey, id }) => {
+                const found = vectors.readiness(context, chatKey, model);
+                return found
+                    ? { chatId: id, ...found, builtAt: new Date(found.builtAt).toISOString() }
+                    : {
+                        chatId: id,
+                        builtAt: null,
+                        algorithmVersion: null,
+                        pending: { new: 0, edited: 0, deleted: 0 },
+                        vectors: { chunks: 0, embedded: 0, current: 0, stale: 0, missing: 0 },
+                    };
+            });
+        },
+        unbuiltGroups: async (key) => {
+            const accountPk = findAccountPk(key);
+            return accountPk === undefined ? [] : vectors.unbuiltGroups(context, accountPk);
+        },
+        embeddedOnlyElsewhere: async (key, { chatId, model }) => {
+            const accountPk = findAccountPk(key);
+            if (accountPk === undefined)
+                return [];
+            const chatKey = chatId === undefined ? undefined : chatKeyOf(key, chatId);
+            if (chatId !== undefined && chatKey === undefined)
+                return [];
+            return vectors.embeddedOnlyElsewhere(context, accountPk, { ...(chatKey === undefined ? {} : { chatKey }), model });
+        },
+        conversationState: async (key, chatId) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? undefined : conversationQueries.stateOf(context, chatKey);
+        },
+        chats: async (key, window) => {
+            const accountKey = findAccountPk(key);
+            return accountKey === undefined
+                ? { items: [], hasMore: false }
+                : chatQueries.listChats(context, accountKey, window);
+        },
+        countChats: async (key, filter = {}) => {
+            const accountKey = findAccountPk(key);
+            return accountKey === undefined ? 0 : chatQueries.countChats(context, accountKey, filter);
+        },
+        countMessages: async (key, chatId, { since } = {}) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? 0 : reads.countMessages(context, chatKey, since);
+        },
+        documentLinkMessages: async (key, chatId) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? [] : documentLinkMessages(context, chatKey);
+        },
+        changes: async (key, chatId, at) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined
+                ? { messages: [], deleted: [] }
+                : reads.changesSince(context, chatKey, Date.parse(at));
+        },
+        messagesWindow: async (key, chatId, window) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? [] : reads.messagesWindow(context, chatKey, window);
+        },
+        messages: async (key, chatId, window) => {
+            const chatKey = chatKeyOf(key, chatId);
+            return chatKey === undefined ? { items: [], hasMore: false } : reads.messagePage(context, chatKey, window);
+        },
+        around: async (key, chatId, messageId, window) => reads.around(context, chatKeyOf(key, chatId), messageId, window),
+        message: async (key, messageId, { chatId } = {}) => {
+            const accountKey = findAccountPk(key);
+            return accountKey === undefined ? undefined : reads.message(context, accountKey, messageId, chatId);
+        },
+        markDeleted: async (key, messageIds, { chatId, among } = {}) => {
+            const accountKey = findAccountPk(key);
+            if (accountKey === undefined || messageIds.length === 0)
+                return 0;
+            let changed = 0;
+            inTransaction(() => {
+                changed = messageWrites.markDeleted(context, accountKey, messageIds, chatId, among);
+            });
+            return changed;
+        },
+        purge: async (key) => {
+            const accountKey = findAccountPk(key);
+            if (accountKey === undefined)
+                return;
+            inTransaction(() => accounts.purgeAccount(context, accountKey));
+        },
+        markChatsLeft: async (key, present) => {
+            const accountKey = findAccountPk(key);
+            if (accountKey === undefined)
+                return 0;
+            let marked = 0;
+            inTransaction(() => {
+                marked = chatQueries.markLeft(context, accountKey, present);
+                writeState(accountKey, completeness.CHAT_LIST_KEY, String(present.length));
+            });
+            return marked;
+        },
+        leftChats: async (key, { clear = false } = {}) => {
+            const accountKey = findAccountPk(key);
+            if (accountKey === undefined)
+                return { chats: 0, messages: 0 };
+            let found = { pks: [], messages: 0 };
+            inTransaction(() => {
+                found = chatQueries.leftChats(context, accountKey);
+                if (clear)
+                    chatQueries.purgeChats(context, found.pks);
+            });
+            return { chats: found.pks.length, messages: found.messages };
+        },
+        search: async (query, { limit, account }) => search.find(context, { text: query, limit, ...(account ? { account } : {}) }),
+        find: async (filter) => (filter.pattern ? findRegex(context, filter) : search.find(context, filter)),
+        rankingDiscussionChats: async (account, chatId) => {
+            const rows = database
+                .prepare("SELECT DISTINCT json_extract(CASE WHEN json_valid(m.metadata) THEN m.metadata ELSE '{}' END,'$.graph.discussionChatId') AS id FROM messages m JOIN chats c ON c.id=m.chat_id JOIN accounts ac ON ac.id=m.account_id WHERE ac.provider=? AND ac.external_id=? AND c.external_id=? AND c.kind='channel' AND m.deleted_at IS NULL AND json_extract(CASE WHEN json_valid(m.metadata) THEN m.metadata ELSE '{}' END,'$.graph.version')=1 AND json_type(CASE WHEN json_valid(m.metadata) THEN m.metadata ELSE '{}' END,'$.graph.discussionChatId')='text' LIMIT 101")
+                .all(account.provider, account.account, chatId);
+            if (rows.length > 100)
+                throw new CliError("validation_error", "too many linked discussion groups — narrow the channel scope");
+            return rows.map(({ id }) => String(id));
+        },
+        rankingEvidence: async (execution, request) => rankingEvidence(context, execution, request),
+        retention: async (key, chatId, options, evidence) => {
+            const accountKey = findAccountPk(key);
+            const chatKey = accountKey === undefined ? undefined : findChatPk(accountKey, chatId);
+            if (accountKey === undefined || chatKey === undefined)
+                throw new CliError("validation_error", "retention needs a stored chat and membership observations");
+            context.database.exec("BEGIN");
+            try {
+                const result = retentionQuery(context, accountKey, chatKey, chatId, options, evidence);
+                context.database.exec("COMMIT");
+                return result;
+            }
+            catch (error) {
+                context.database.exec("ROLLBACK");
+                throw error;
+            }
+        },
+        adminStatisticsQuery: async (execution, request) => adminStatisticsQuery(context, execution, request),
+        rankQuery: async (execution, request) => rankQuery(context, execution, request),
+        directReplies: async (parents, limit) => search.directReplies(context, parents, limit),
+        matchQuery: async (execution) => lucene.matchQuery(context, mailIndexed(execution)),
+        conversationEligibility: async (execution) => conversationEligibility(context, execution),
+        countQuery: async (execution, by) => lucene.countQuery(context, mailIndexed(execution), by),
+        matchWords: async (query, scope, options) => words.matchWords(context, query, scope, options),
+        matchSubstring: async (query, scope, options) => words.matchSubstring(context, query, scope, options),
+        attachmentKinds: async () => database
+            .prepare("SELECT DISTINCT kind FROM attachments ORDER BY kind")
+            .all()
+            .map((row) => String(row.kind)),
+        matchFilters: async (scope, options) => words.matchFilters(context, scope, options),
+        accounts: async () => accounts.heldAccounts(context),
+        storedAccounts: async () => accounts.storedAccounts(context),
+        storedAccount: async (key) => accounts.storedAccount(context, key),
+        accountName: async (key) => accounts.accountName(context, key),
+        chatCompleteness: async (key, chatIds) => {
+            const accountKey = accounts.findAccountPk(context, key);
+            return accountKey === undefined ? [] : completeness.chatCompleteness(context, accountKey, chatIds);
+        },
+        knownTerms: async (terms) => words.knownTerms(context, terms),
+        termCandidates: async (trigrams, lengths) => words.termCandidates(context, trigrams, lengths),
+        searchIndexState: async () => searchIndex.searchIndexState(database),
+        fillSearchIndex: async ({ until } = {}) => searchIndex.fillSearchIndex(database, { now: context.now, ...(until ? { until } : {}) }),
+        stemsState: async () => stems.stemsState(database),
+        fillStems: async ({ until } = {}) => stems.fillStems(database, { now: context.now, ...(until ? { until } : {}) }),
+        stemmers: async () => stems.savedStemmers(database),
+        saveStemmers: async (stemmers) => inTransaction(() => stems.saveStoreStemmers(database, stemmers, context.now())),
+        accountGroups: async () => accountGroupsOf(database),
+        profiles: async () => profilesOf(database),
+        saveProfile: async (app, profile, key) => inTransaction(() => saveProfile(database, app, profile, key, context.now())),
+        saveAccountGroup: async (name, members) => inTransaction(() => {
+            const groups = accountGroupsOf(database);
+            if (members === undefined)
+                delete groups[name];
+            else
+                groups[name] = members;
+            database
+                .prepare("INSERT OR REPLACE INTO store_settings (key, value, at) VALUES (?, ?, ?)")
+                .run(ACCOUNT_GROUPS, JSON.stringify(groups), context.now());
+        }),
+        contacts: async (key, options) => {
+            const accountKey = findAccountPk(key);
+            if (accountKey === undefined)
+                return { items: [], hasMore: false };
+            return identities.contacts(context, accountKey, key, options);
+        },
+        countContacts: async (key, { query } = {}) => {
+            const accountKey = findAccountPk(key);
+            return accountKey === undefined ? 0 : identities.countContacts(context, accountKey, key, query);
+        },
+        refreshRecency: async (key) => {
+            const accountKey = findAccountPk(key);
+            if (accountKey !== undefined)
+                identities.refreshRecency(context, accountKey);
+        },
+        savePeople: async (key, people) => inTransaction(() => {
+            const accountKey = accountPk(key);
+            for (const person of people)
+                identityPk(accountKey, key.provider, person.id, person.name, person);
+        }),
+        people: async (provider, options = {}) => identities.people(context, provider, options),
+        counterTargets: async (execution, options) => counterTargets(context, execution, options),
+        counterStates: async (key, chatId, messageId, options) => {
+            const row = database
+                .prepare("SELECT m.id AS pk FROM messages m JOIN chats c ON c.id=m.chat_id JOIN accounts ac ON ac.id=m.account_id WHERE ac.provider=? AND ac.external_id=? AND c.external_id=? AND m.external_id=? AND m.deleted_at IS NULL")
+                .get(key.provider, key.account, chatId, messageId);
+            return row ? counterStates(context, Number(row.pk), options.now, options.maxAge) : [];
+        },
+        updateCounterObservations: async (key, chatId, messageId, observations) => {
+            let updated = 0;
+            inTransaction(() => {
+                const row = database
+                    .prepare("SELECT m.id AS pk FROM messages m JOIN chats c ON c.id=m.chat_id JOIN accounts ac ON ac.id=m.account_id WHERE ac.provider=? AND ac.external_id=? AND c.external_id=? AND m.external_id=? AND m.deleted_at IS NULL")
+                    .get(key.provider, key.account, chatId, messageId);
+                if (row)
+                    updated = applyCounterObservations(context, Number(row.pk), observations);
+            });
+            return updated;
+        },
+        saveReactions: async (key, chatId, messageId, reactions, options = {}) => {
+            const accountKey = findAccountPk(key);
+            const chatKey = accountKey === undefined ? undefined : findChatPk(accountKey, chatId);
+            if (chatKey === undefined)
+                return false;
+            let saved = false;
+            inTransaction(() => {
+                saved = messageWrites.saveReactions(context, chatKey, messageId, reactions);
+                const row = database
+                    .prepare("SELECT id AS pk FROM messages WHERE chat_id=? AND external_id=? AND deleted_at IS NULL")
+                    .get(chatKey, messageId);
+                if (row && options.observation)
+                    applyCounterObservations(context, Number(row.pk), { reactions: options.observation });
+            });
+            return saved;
+        },
+        markRange: async (key, chatId, from, to) => {
+            let merged = { from, to };
+            inTransaction(() => {
+                merged = ranges.markRange(context, chatPkFor(accountPk(key), chatId), from, to);
+            });
+            return merged;
+        },
+        senderStats: async (key, senderId) => {
+            const accountKey = findAccountPk(key);
+            return accountKey === undefined ? [] : reads.senderStats(context, accountKey, key.provider, senderId);
+        },
+        personNames: async (key, personId) => {
+            const accountKey = findAccountPk(key);
+            return accountKey === undefined ? [] : identities.namesOf(context, accountKey, key.provider, personId);
+        },
+        chatStats: async (key, chatId) => {
+            const accountKey = findAccountPk(key);
+            return accountKey === undefined ? [] : reads.chatStats(context, accountKey, chatId);
+        },
+        chatActivity: async (key, since) => {
+            const accountKey = findAccountPk(key);
+            return accountKey === undefined ? { chats: [], storedAt: null } : reads.chatActivity(context, accountKey, since);
+        },
+        ranges: async (key, chatId) => {
+            const accountKey = findAccountPk(key);
+            const chatKey = accountKey === undefined ? undefined : findChatPk(accountKey, chatId);
+            return chatKey === undefined ? [] : ranges.ranges(context, chatKey);
+        },
+        chatMetadata: async (key, chatId) => metadataQueries.metadata(context, key, chatId),
+        saveChatMetadata: async (key, entry) => metadataQueries.saveMetadata(context, key, entry),
+        replaceAutoTags: async (key, chatId, algorithm, matches) => inTransaction(() => metadataQueries.replaceAutoTags(context, key, chatId, algorithm, matches)),
+        privateContact: async (key, person) => privatePeople.privateContact(context, key, person),
+        setContactAlias: async (key, person, alias) => {
+            let result;
+            inTransaction(() => {
+                result = privatePeople.setAlias(context, key, person, alias);
+            });
+            return result;
+        },
+        addContactNote: async (key, person, text) => {
+            let added;
+            inTransaction(() => {
+                added = privatePeople.addNote(context, key, person, text);
+            });
+            return added;
+        },
+        contactNote: async (key, person, id) => privatePeople.note(context, key, person, id),
+        editContactNote: async (key, person, id, text, revision) => {
+            let edited;
+            inTransaction(() => {
+                edited = privatePeople.editNote(context, key, person, id, text, revision);
+            });
+            return edited;
+        },
+        removeContactNote: async (key, person, id) => privatePeople.removeNote(context, key, person, id),
+        addTags: async (key, target, list) => {
+            let added = [];
+            inTransaction(() => {
+                added = tagQueries.addTags(context, tagQueries.targetThing(context, key, target), list);
+            });
+            return added;
+        },
+        removeTags: async (key, target, list, source) => {
+            let removed = [];
+            inTransaction(() => {
+                removed = tagQueries.removeTags(context, tagQueries.targetThing(context, key, target), list, source);
+            });
+            return removed;
+        },
+        tags: async (key, filter = {}) => tagQueries.tagsOf(context, key, filter),
+        recordSearch: async (run, { saved } = {}) => inTransaction(() => searchRecords.recordRun(context, run, saved)),
+        saveSearch: async (name, search, { replace = false } = {}) => {
+            let saved;
+            inTransaction(() => {
+                saved = searchRecords.saveSearch(context, name, search, replace);
+            });
+            return saved;
+        },
+        storedSearch: async (reference) => searchRecords.findSearch(context, reference),
+        savedSearches: async () => searchRecords.savedSearches(context),
+        searchHistory: async (limit) => searchRecords.searchHistory(context, limit),
+        deleteSearch: async (reference) => {
+            let deleted;
+            inTransaction(() => {
+                deleted = searchRecords.deleteSearch(context, reference);
+            });
+            return deleted;
+        },
+        clearSearchHistory: async () => {
+            let cleared = 0;
+            inTransaction(() => {
+                cleared = searchRecords.clearHistory(context);
+            });
+            return cleared;
+        },
+        // Built on first use: an area that prepares its statements must not stop the store opening for the rest.
+        get botUpdates() {
+            botUpdates ??= botUpdateStoreOver(context, pruneLogsWhenDue);
+            return botUpdates;
+        },
+        get involvements() {
+            involvements ??= involvementStoreOver(context);
+            return involvements;
+        },
+        get tasks() {
+            tasks ??= taskStoreOver(context);
+            return tasks;
+        },
+        get knowledge() {
+            knowledge ??= knowledgeStoreOver(context);
+            return knowledge;
+        },
+        get notes() {
+            notes ??= { ...notesStoreOver(context), ...noteSearchOver(context) };
+            return notes;
+        },
+        get catalog() {
+            catalog ??= catalogStoreOver(context);
+            return catalog;
+        },
+        get facts() {
+            facts ??= factsStoreOver(context);
+            return facts;
+        },
+        get memories() {
+            memories ??= memoriesStoreOver(context);
+            return memories;
+        },
+        get proposedActions() {
+            proposedActions ??= proposedActionsStoreOver(context);
+            return proposedActions;
+        },
+        get agentActions() {
+            agentActions ??= agentActionsStoreOver(context, pruneLogsWhenDue);
+            return agentActions;
+        },
+        get meetings() {
+            meetings ??= meetingStoreOver(context);
+            return meetings;
+        },
+        get mail() {
+            return mailOf();
+        },
+        forAccount: (key) => bindAccount(store, key, {
+            accountPk: () => findAccountPk(key),
+            attachmentAccount: (attachmentPk) => attachmentTexts.accountOf(context, attachmentPk),
+        }),
+        close: async () => database.close(),
+    };
+    return store;
+};
+export { accountGroupsOf } from "./sqlite/account-groups.js";
+export { resetAttachmentWords, } from "./sqlite/attachment-texts.js";
+export { pendingNormalization } from "./sqlite/backfill.js";
+export { CHAT_LIST_KEY, fetchedKey, historyStartKey } from "./sqlite/completeness.js";
+export { messageOfEmail, threadConversation } from "./sqlite/emails.js";
+export { orphanPointers } from "./sqlite/entity-types.js";
+export { involvementStoreOver } from "./sqlite/involvements.js";
+export { drainNoteIndex, noteIndexState, resetNoteIndex } from "./sqlite/note-index.js";
+export { fillSearchIndex, resetSearchIndex, searchIndexState } from "./sqlite/search-index.js";
+export { fillStems, resetStems, savedStemmers, stemmerCache, stemmersOrigin, stemsState, } from "./sqlite/stems.js";
+export { inboxKey } from "./sqlite/tasks.js";
+//# sourceMappingURL=store.js.map