@wirecat/cli-core 0.279.0 → 0.281.0

package/dist/store/store.js

@@ -30,16 +30,14 @@ import { factsStoreOver } from "./sqlite/facts.js";
 import { fillFolded } from "./sqlite/folded.js";
 import * as identities from "./sqlite/identities.js";
 import * as imageChunks from "./sqlite/image-chunks.js";
-import { drainInvolvementQueue } from "./sqlite/involvement-queue.js";
 import { involvementStoreOver } from "./sqlite/involvements.js";
 import { knowledgeStoreOver } from "./sqlite/knowledge.js";
-import { findRegex } from "./sqlite/legacy-regex.js";
 import { logPruner } from "./sqlite/log-pruning.js";
 import * as lucene from "./sqlite/lucene.js";
 import { meetingVectorsOver } from "./sqlite/meeting-vectors.js";
 import { meetingStoreOver } from "./sqlite/meetings.js";
 import { memoriesStoreOver } from "./sqlite/memories.js";
-import { confirmMention, drainMentions, listMentions, mentionsState, rebuildMentions, rejectMention, showMention, } from "./sqlite/mentions.js";
+import { confirmMention, listMentions, mentionsState, rebuildMentions, rejectMention, showMention, } from "./sqlite/mentions.js";
 import * as messageWrites from "./sqlite/messages.js";
 import { noteSearchOver } from "./sqlite/note-search.js";
 import { notesStoreOver } from "./sqlite/notes.js";
@@ -58,9 +56,11 @@ import * as search from "./sqlite/search.js";
 import * as searchIndex from "./sqlite/search-index.js";
 import * as searchRecords from "./sqlite/searches.js";
 import * as stems from "./sqlite/stems.js";
+import { fillStems, followDefaultStemmers } from "./sqlite/stems-fill.js";
 import * as sync from "./sqlite/sync.js";
 import * as tagQueries from "./sqlite/tags.js";
 import { linkInbox, taskStoreOver } from "./sqlite/tasks.js";
+import { read, write } from "./sqlite/transaction.js";
 import * as transcripts from "./sqlite/transcripts.js";
 import * as vectors from "./sqlite/vectors.js";
 import * as words from "./sqlite/words.js";
@@ -85,11 +85,11 @@ export const openStore = async ({ path, env, now = Date.now, command } = {}) =>
         const pending = pendingNormalization(database);
         if (pending > 0 && pending <= BACKFILL_ON_OPEN)
             backfillNormalized(database);
-        stems.followDefaultStemmers(database, now);
+        followDefaultStemmers(database, now);
         const stemming = stems.stemsState(database);
         if (stemming?.cause === "building" &&
             stemming.watermark - stemming.filledThrough + stemming.pending <= BACKFILL_ON_OPEN) {
-            stems.fillStems(database, { now });
+            fillStems(database, { now });
         }
         logPruner(database, now)();
     }
@@ -103,20 +103,6 @@ const storeOver = (context) => {
     const { database } = context;
     const stemmerFor = stems.stemmerCache();
     const pruneLogsWhenDue = logPruner(database, context.now);
-    const inTransaction = (body) => {
-        database.exec("BEGIN IMMEDIATE");
-        try {
-            body();
-            stems.drainStems(database, stemmerFor);
-            drainMentions(database, context.now());
-            drainInvolvementQueue(database);
-            database.exec("COMMIT");
-        }
-        catch (error) {
-            database.exec("ROLLBACK");
-            throw error;
-        }
-    };
     const mailIndexed = (execution) => {
         if (execution.corpus === "mail")
             drainEmailIndex(database, stemmerFor);
@@ -157,12 +143,12 @@ const storeOver = (context) => {
             linkInbox(database, `${key.provider}:${key.account}`, pk);
             return pk;
         },
-        saveChats: async (key, list) => inTransaction(() => {
+        saveChats: async (key, list) => write(database, () => {
             const accountKey = accountPk(key);
             for (const chat of list)
                 chatQueries.upsertChat(context, accountKey, chat);
         }),
-        saveMessages: async (key, chatId, messages, { via, seenAt }) => inTransaction(() => {
+        saveMessages: async (key, chatId, messages, { via, seenAt }) => write(database, () => {
             const accountKey = accountPk(key);
             const chatKey = chatPkFor(accountKey, chatId);
             for (const message of messages) {
@@ -170,10 +156,10 @@ const storeOver = (context) => {
                     upsertMessage(key, accountKey, chatKey, message, via, seenAt);
             }
         }),
-        saveMembers: async (key, chatId, memberIds) => inTransaction(() => writeMembers(key, accountPk(key), chatId, memberIds)),
+        saveMembers: async (key, chatId, memberIds) => write(database, () => writeMembers(key, accountPk(key), chatId, memberIds)),
         saveRoster: async (key, chatId, read) => {
             let change = { joined: [], gone: [], changed: [] };
-            inTransaction(() => {
+            write(database, () => {
                 const accountKey = accountPk(key);
                 change = roster.saveRoster(context, accountKey, key.provider, chatPkFor(accountKey, chatId), read);
             });
@@ -195,7 +181,7 @@ const storeOver = (context) => {
                 ? []
                 : roster.profileRevisionsOf(context, chatKey, since === undefined ? undefined : Date.parse(since));
         },
-        trackMembers: async (key, chatId, tracked) => inTransaction(() => roster.setTracked(context, chatPkFor(accountPk(key), chatId), tracked)),
+        trackMembers: async (key, chatId, tracked) => write(database, () => roster.setTracked(context, chatPkFor(accountPk(key), chatId), tracked)),
         trackedChats: async (key) => {
             const accountKey = findAccountPk(key);
             return accountKey === undefined ? [] : roster.trackedChats(context, accountKey);
@@ -209,18 +195,10 @@ const storeOver = (context) => {
         personSeenInAccounts: async (uid, accountIds) => personSeenInAccounts(context, uid, accountIds),
         personByUid: async (uid) => personLinks.personByUid(context, uid),
         linkIdentities: async (person, other, options) => {
-            let linked;
-            inTransaction(() => {
-                linked = personLinks.linkIdentities(context, person, other, options);
-            });
-            return linked;
+            return write(database, () => personLinks.linkIdentities(context, person, other, options));
         },
         unlinkIdentity: async (identity, options) => {
-            let alone;
-            inTransaction(() => {
-                alone = personLinks.unlinkIdentity(context, identity, options);
-            });
-            return alone;
+            return write(database, () => personLinks.unlinkIdentity(context, identity, options));
         },
         mentioning: async (key, person, limit) => {
             const accountKey = findAccountPk(key);
@@ -236,8 +214,8 @@ const storeOver = (context) => {
             const accountKey = findAccountPk(key);
             return accountKey === undefined ? undefined : sync.syncStateOf(context, accountKey, name);
         },
-        setSyncState: async (key, name, value) => inTransaction(() => writeState(accountPk(key), name, value)),
-        applyDelta: async (key, { chats = [], people = [], members = new Map(), state = {} }) => inTransaction(() => {
+        setSyncState: async (key, name, value) => write(database, () => writeState(accountPk(key), name, value)),
+        applyDelta: async (key, { chats = [], people = [], members = new Map(), state = {} }) => write(database, () => {
             const accountKey = accountPk(key);
             for (const chat of chats)
                 chatQueries.upsertChat(context, accountKey, chat);
@@ -255,7 +233,7 @@ const storeOver = (context) => {
         },
         claim: async (key, chatId, anchor, holder, forMs) => {
             let taken = false;
-            inTransaction(() => {
+            write(database, () => {
                 taken = sync.claim(context, chatPkFor(accountPk(key), chatId), anchor, holder, forMs);
             });
             return taken;
@@ -274,7 +252,7 @@ const storeOver = (context) => {
         keepTranscript: async (key, chatId, messageId, text, source) => {
             if (text.trim() === "")
                 return;
-            inTransaction(() => transcripts.keepTranscript(context, chatPkFor(accountPk(key), chatId), messageId, text, source));
+            write(database, () => transcripts.keepTranscript(context, chatPkFor(accountPk(key), chatId), messageId, text, source));
         },
         fileAttachments: async (key, { chatId, ...page }) => {
             const accountKey = findAccountPk(key);
@@ -297,7 +275,7 @@ const storeOver = (context) => {
         localPathOf: async (attachmentPk) => attachmentTexts.localPathOf(context, attachmentPk),
         keepAttachmentText: async (attachmentPk, entry) => {
             let kept = false;
-            inTransaction(() => {
+            write(database, () => {
                 kept = attachmentTexts.keepText(context, attachmentPk, entry);
             });
             return kept;
@@ -307,7 +285,7 @@ const storeOver = (context) => {
             if (chatKey === undefined || files.length === 0)
                 return 0;
             let kept = 0;
-            inTransaction(() => {
+            write(database, () => {
                 kept = attachmentRows.keepDownloads(context, chatKey, messageId, files);
             });
             return kept;
@@ -377,7 +355,7 @@ const storeOver = (context) => {
                 throw new CliError("validation_error", `the answer was not stored: ${batch} is not a batch of this account`);
             }
             let stored = 0;
-            inTransaction(() => {
+            write(database, () => {
                 stored = batches.saveAnswers(context, batch, answer);
             });
             return { chat, stored };
@@ -387,7 +365,7 @@ const storeOver = (context) => {
             if (chatKey === undefined)
                 return 0;
             let cleared = 0;
-            inTransaction(() => {
+            write(database, () => {
                 cleared = batches.clearAnswers(context, chatKey, model);
             });
             return cleared;
@@ -418,7 +396,7 @@ const storeOver = (context) => {
             return chatKey === undefined ? [] : vectors.chunksToEmbed(context, chatKey, model, options);
         },
         saveVectors: async (model, dims, rows) => {
-            inTransaction(() => vectors.saveVectors(context, model, dims, rows));
+            write(database, () => vectors.saveVectors(context, model, dims, rows));
         },
         imagesToDescribe: async (key, chatId, window) => {
             const chatKey = chatKeyOf(key, chatId);
@@ -429,7 +407,7 @@ const storeOver = (context) => {
             const chatKey = chatKeyOf(key, chatId);
             if (accountPk === undefined || chatKey === undefined)
                 return;
-            inTransaction(() => imageChunks.saveImageChunks(context, accountPk, chatKey, chunks, keep));
+            write(database, () => imageChunks.saveImageChunks(context, accountPk, chatKey, chunks, keep));
         },
         embeddedHashes: async (model, hashes) => imageChunks.embeddedHashes(context, model, hashes),
         nearestImages: async (key, { chatId, model, since, limit, query }) => {
@@ -481,7 +459,7 @@ const storeOver = (context) => {
             const accountPk = findAccountPk(key);
             if (accountPk === undefined)
                 return;
-            inTransaction(() => {
+            write(database, () => {
                 vectors.saveVectors(context, model, dims, pictures.map(({ hash, vector }) => ({ hash, vector })));
                 imageChunks.savePictureChunks(context, accountPk, pictures);
             });
@@ -511,7 +489,7 @@ const storeOver = (context) => {
             if (chatKey === undefined)
                 return 0;
             let cleared = 0;
-            inTransaction(() => {
+            write(database, () => {
                 cleared = vectors.clearVectors(context, chatKey, model);
             });
             return cleared;
@@ -672,21 +650,13 @@ const storeOver = (context) => {
                     owner = { organization: Number(found.id) };
                 }
             }
-            let decision;
-            inTransaction(() => {
-                decision = confirmMention(context, accountKey, id, owner, by);
-            });
-            return decision;
+            return write(database, () => confirmMention(context, accountKey, id, owner, by));
         },
         rejectMention: async (key, id, by) => {
             const accountKey = findAccountPk(key);
             if (accountKey === undefined)
                 throw new CliError("not_found", `no mention ${id} in this account`);
-            let decision;
-            inTransaction(() => {
-                decision = rejectMention(context, accountKey, id, by);
-            });
-            return decision;
+            return write(database, () => rejectMention(context, accountKey, id, by));
         },
         rebuildMentions: async (options = {}) => rebuildMentions(database, { now: context.now(), ...options }),
         mentionsState: async () => mentionsState(database),
@@ -714,7 +684,7 @@ const storeOver = (context) => {
             if (accountKey === undefined || messageIds.length === 0)
                 return 0;
             let changed = 0;
-            inTransaction(() => {
+            write(database, () => {
                 changed = messageWrites.markDeleted(context, accountKey, messageIds, chatId, among);
             });
             return changed;
@@ -723,14 +693,14 @@ const storeOver = (context) => {
             const accountKey = findAccountPk(key);
             if (accountKey === undefined)
                 return;
-            inTransaction(() => accounts.purgeAccount(context, accountKey));
+            write(database, () => accounts.purgeAccount(context, accountKey));
         },
         markChatsLeft: async (key, present) => {
             const accountKey = findAccountPk(key);
             if (accountKey === undefined)
                 return 0;
             let marked = 0;
-            inTransaction(() => {
+            write(database, () => {
                 marked = chatQueries.markLeft(context, accountKey, present);
                 writeState(accountKey, completeness.CHAT_LIST_KEY, String(present.length));
             });
@@ -741,7 +711,7 @@ const storeOver = (context) => {
             if (accountKey === undefined)
                 return { chats: 0, messages: 0 };
             let found = { pks: [], messages: 0 };
-            inTransaction(() => {
+            write(database, () => {
                 found = chatQueries.leftChats(context, accountKey);
                 if (clear)
                     chatQueries.purgeChats(context, found.pks);
@@ -749,7 +719,7 @@ const storeOver = (context) => {
             return { chats: found.pks.length, messages: found.messages };
         },
         search: async (query, { limit, account }) => search.find(context, { text: query, limit, ...(account ? { account } : {}) }),
-        find: async (filter) => (filter.pattern ? findRegex(context, filter) : search.find(context, filter)),
+        find: async (filter) => search.find(context, filter),
         rankingDiscussionChats: async (account, chatId) => {
             const rows = database
                 .prepare("SELECT DISTINCT json_extract(CASE WHEN json_valid(m.metadata) THEN m.metadata ELSE '{}' END,'$.graph.discussionChatId') AS id FROM messages m JOIN chats c ON c.id=m.chat_id JOIN accounts ac ON ac.id=m.account_id WHERE ac.provider=? AND ac.external_id=? AND c.external_id=? AND c.kind='channel' AND m.deleted_at IS NULL AND json_extract(CASE WHEN json_valid(m.metadata) THEN m.metadata ELSE '{}' END,'$.graph.version')=1 AND json_type(CASE WHEN json_valid(m.metadata) THEN m.metadata ELSE '{}' END,'$.graph.discussionChatId')='text' LIMIT 101")
@@ -764,16 +734,7 @@ const storeOver = (context) => {
             const chatKey = accountKey === undefined ? undefined : findChatPk(accountKey, chatId);
             if (accountKey === undefined || chatKey === undefined)
                 throw new CliError("validation_error", "retention needs a stored chat and membership observations");
-            context.database.exec("BEGIN");
-            try {
-                const result = retentionQuery(context, accountKey, chatKey, chatId, options, evidence);
-                context.database.exec("COMMIT");
-                return result;
-            }
-            catch (error) {
-                context.database.exec("ROLLBACK");
-                throw error;
-            }
+            return read(database, () => retentionQuery(context, accountKey, chatKey, chatId, options, evidence));
         },
         adminStatisticsQuery: async (execution, request) => adminStatisticsQuery(context, execution, request),
         rankQuery: async (execution, request) => rankQuery(context, execution, request),
@@ -781,13 +742,6 @@ const storeOver = (context) => {
         matchQuery: async (execution) => lucene.matchQuery(context, mailIndexed(execution)),
         conversationEligibility: async (execution) => conversationEligibility(context, execution),
         countQuery: async (execution, by) => lucene.countQuery(context, mailIndexed(execution), by),
-        matchWords: async (query, scope, options) => words.matchWords(context, query, scope, options),
-        matchSubstring: async (query, scope, options) => words.matchSubstring(context, query, scope, options),
-        attachmentKinds: async () => database
-            .prepare("SELECT DISTINCT kind FROM attachments ORDER BY kind")
-            .all()
-            .map((row) => String(row.kind)),
-        matchFilters: async (scope, options) => words.matchFilters(context, scope, options),
         accounts: async () => accounts.heldAccounts(context),
         storedAccounts: async () => accounts.storedAccounts(context),
         storedAccount: async (key) => accounts.storedAccount(context, key),
@@ -801,13 +755,13 @@ const storeOver = (context) => {
         searchIndexState: async () => searchIndex.searchIndexState(database),
         fillSearchIndex: async ({ until } = {}) => searchIndex.fillSearchIndex(database, { now: context.now, ...(until ? { until } : {}) }),
         stemsState: async () => stems.stemsState(database),
-        fillStems: async ({ until } = {}) => stems.fillStems(database, { now: context.now, ...(until ? { until } : {}) }),
+        fillStems: async ({ until } = {}) => fillStems(database, { now: context.now, ...(until ? { until } : {}) }),
         stemmers: async () => stems.savedStemmers(database),
-        saveStemmers: async (stemmers) => inTransaction(() => stems.saveStoreStemmers(database, stemmers, context.now())),
+        saveStemmers: async (stemmers) => write(database, () => stems.saveStoreStemmers(database, stemmers, context.now())),
         accountGroups: async () => accountGroupsOf(database),
         profiles: async () => profilesOf(database),
-        saveProfile: async (app, profile, key) => inTransaction(() => saveProfile(database, app, profile, key, context.now())),
-        saveAccountGroup: async (name, members) => inTransaction(() => {
+        saveProfile: async (app, profile, key) => write(database, () => saveProfile(database, app, profile, key, context.now())),
+        saveAccountGroup: async (name, members) => write(database, () => {
             const groups = accountGroupsOf(database);
             if (members === undefined)
                 delete groups[name];
@@ -832,7 +786,7 @@ const storeOver = (context) => {
             if (accountKey !== undefined)
                 identities.refreshRecency(context, accountKey);
         },
-        savePeople: async (key, people) => inTransaction(() => {
+        savePeople: async (key, people) => write(database, () => {
             const accountKey = accountPk(key);
             for (const person of people)
                 identityPk(accountKey, key.provider, person.id, person.name, person);
@@ -847,7 +801,7 @@ const storeOver = (context) => {
         },
         updateCounterObservations: async (key, chatId, messageId, observations) => {
             let updated = 0;
-            inTransaction(() => {
+            write(database, () => {
                 const row = database
                     .prepare("SELECT m.id AS pk FROM messages m JOIN chats c ON c.id=m.chat_id JOIN accounts ac ON ac.id=m.account_id WHERE ac.provider=? AND ac.external_id=? AND c.external_id=? AND m.external_id=? AND m.deleted_at IS NULL")
                     .get(key.provider, key.account, chatId, messageId);
@@ -862,7 +816,7 @@ const storeOver = (context) => {
             if (chatKey === undefined)
                 return false;
             let saved = false;
-            inTransaction(() => {
+            write(database, () => {
                 saved = messageWrites.saveReactions(context, chatKey, messageId, reactions);
                 const row = database
                     .prepare("SELECT id AS pk FROM messages WHERE chat_id=? AND external_id=? AND deleted_at IS NULL")
@@ -874,7 +828,7 @@ const storeOver = (context) => {
         },
         markRange: async (key, chatId, from, to) => {
             let merged = { from, to };
-            inTransaction(() => {
+            write(database, () => {
                 merged = ranges.markRange(context, chatPkFor(accountPk(key), chatId), from, to);
             });
             return merged;
@@ -904,67 +858,47 @@ const storeOver = (context) => {
         },
         chatMetadata: async (key, chatId) => metadataQueries.metadata(context, key, chatId),
         saveChatMetadata: async (key, entry) => metadataQueries.saveMetadata(context, key, entry),
-        replaceAutoTags: async (key, chatId, algorithm, matches) => inTransaction(() => metadataQueries.replaceAutoTags(context, key, chatId, algorithm, matches)),
+        replaceAutoTags: async (key, chatId, algorithm, matches) => write(database, () => metadataQueries.replaceAutoTags(context, key, chatId, algorithm, matches)),
         privateContact: async (key, person) => privatePeople.privateContact(context, key, person),
         setContactAlias: async (key, person, alias) => {
-            let result;
-            inTransaction(() => {
-                result = privatePeople.setAlias(context, key, person, alias);
-            });
-            return result;
+            return write(database, () => privatePeople.setAlias(context, key, person, alias));
         },
         addContactNote: async (key, person, text) => {
-            let added;
-            inTransaction(() => {
-                added = privatePeople.addNote(context, key, person, text);
-            });
-            return added;
+            return write(database, () => privatePeople.addNote(context, key, person, text));
         },
         contactNote: async (key, person, id) => privatePeople.note(context, key, person, id),
         editContactNote: async (key, person, id, text, revision) => {
-            let edited;
-            inTransaction(() => {
-                edited = privatePeople.editNote(context, key, person, id, text, revision);
-            });
-            return edited;
+            return write(database, () => privatePeople.editNote(context, key, person, id, text, revision));
         },
         removeContactNote: async (key, person, id) => privatePeople.removeNote(context, key, person, id),
         addTags: async (key, target, list) => {
             let added = [];
-            inTransaction(() => {
+            write(database, () => {
                 added = tagQueries.addTags(context, tagQueries.targetThing(context, key, target), list);
             });
             return added;
         },
         removeTags: async (key, target, list, source) => {
             let removed = [];
-            inTransaction(() => {
+            write(database, () => {
                 removed = tagQueries.removeTags(context, tagQueries.targetThing(context, key, target), list, source);
             });
             return removed;
         },
         tags: async (key, filter = {}) => tagQueries.tagsOf(context, key, filter),
-        recordSearch: async (run, { saved } = {}) => inTransaction(() => searchRecords.recordRun(context, run, saved)),
+        recordSearch: async (run, { saved } = {}) => write(database, () => searchRecords.recordRun(context, run, saved)),
         saveSearch: async (name, search, { replace = false } = {}) => {
-            let saved;
-            inTransaction(() => {
-                saved = searchRecords.saveSearch(context, name, search, replace);
-            });
-            return saved;
+            return write(database, () => searchRecords.saveSearch(context, name, search, replace));
         },
         storedSearch: async (reference) => searchRecords.findSearch(context, reference),
         savedSearches: async () => searchRecords.savedSearches(context),
         searchHistory: async (limit, offset) => searchRecords.searchHistory(context, limit, offset),
         deleteSearch: async (reference) => {
-            let deleted;
-            inTransaction(() => {
-                deleted = searchRecords.deleteSearch(context, reference);
-            });
-            return deleted;
+            return write(database, () => searchRecords.deleteSearch(context, reference));
         },
         clearSearchHistory: async () => {
             let cleared = 0;
-            inTransaction(() => {
+            write(database, () => {
                 cleared = searchRecords.clearHistory(context);
             });
             return cleared;
@@ -1039,6 +973,7 @@ export { drainNoteIndex, noteIndexState, resetNoteIndex } from "./sqlite/note-in
 export { queryFilter } from "./sqlite/query-filter.js";
 export { AGENDA_COLUMNS } from "./sqlite/reads.js";
 export { fillSearchIndex, resetSearchIndex, searchIndexState } from "./sqlite/search-index.js";
-export { fillStems, resetStems, savedStemmers, stemmerCache, stemmersOrigin, stemsState, } from "./sqlite/stems.js";
+export { savedStemmers, stemmerCache, stemmersOrigin, stemsState, } from "./sqlite/stems.js";
+export { fillStems, resetStems } from "./sqlite/stems-fill.js";
 export { inboxKey } from "./sqlite/tasks.js";
 //# sourceMappingURL=store.js.map