@wirecat/cli-core 0.504.0 → 0.506.0

package/CHANGELOG.md

@@ -7,6 +7,63 @@ shape of this file.
 
 ## Unreleased
 
+## 0.506.0 — 31.10.2026
+
+### Added
+
+- **`QueriesStore.matchingPks`, `AccountVectors.scoreMessages` and `MeaningLane`'s fourth argument `within`**, which
+  lists a filter's messages once meaning can run: together they rank a filter's own messages. `fieldsOfQuery` and
+  `meaningWords` live in `src/search/lucene/meaning-parts.ts`, shared by notes and messages.
+
+- **`MailStore.saveThreads(inputs, states?)`.** Saves several threads and sync state (`{ accountId, name, value }`,
+  as `setSyncState` saves it) in one write transaction: all of it commits or none of it does. Like
+  `EventStore.saveSlots`, it lets a sync commit a slice of work with its resume point.
+
+### Changed — may break callers
+
+- **`search messages --discover` searches by meaning when the query uses `-`, `NOT`, `OR`, groups, phrases or
+  `exact:`.** Such a query used to run strict search alone, with no meaning hits. Now its words are still matched by
+  the whole query, as strict search does (`discovery.method: "lexical-strict"`), and the messages nearest in meaning
+  to its wanted words join them; every hit found by meaning, and every reply and follow-up discovery adds, matches
+  every field of the query and every excluded word, as notes do since 0.496.0. A field joined to the words by `OR`
+  narrows nothing. A query with no word to embed (`date:today`, `Helix*`) stays strict. On the owner's store, 6
+  such queries went from 0 meaning hits to 10–17 of 20, with no hit breaking the query.
+### Fixed
+
+- **Search by meaning inside a filter ranks the filter's own messages.** With fields that hold at most 100,000
+  messages (`RANK_WITHIN_MESSAGES`), `--discover` scores every one of them that has a vector, instead of taking the
+  100 nearest messages overall and dropping those outside the filter. On the owner's store, `chat:<id>` with a chat of
+  1,000 messages went from 3 meaning candidates to 100 and from 9 hits to 20. A larger filter, or one the store
+  cannot list in one query (a preset, a pattern), keeps the old way.
+- **`--discover` works on a read-only store.** Its follow-up step created a temporary table to count rare words,
+  which a read-only connection refuses; it now counts them with a bounded query.
+- **`mail sync` no longer holds the store's one writer lock for each email in turn.** It saves fetched mail in slices of
+  50 emails, each with the resume point in the same transaction, and pauses 100 ms between slices, so `memo sync` or
+  another process next to it can take the lock instead of failing with "database is locked". A run that fails resumes
+  after the last committed slice.
+- **A long fill or rebuild in one process no longer makes another process's write fail with "database is locked".**
+  `writeBatch` pauses 2 ms after each batch it commits. Without the pause, the next batch took the lock again within
+  microseconds, and a write waiting in another process (a command run during `store rebuild` or a migration's fill)
+  could miss every gap until its 5 s wait ran out.
+- **A forgotten memory's or fact's number is never given to a new one.** SQLite gave the next row the highest id + 1,
+  so after `memories.remove` or `facts.remove` of the newest record, the next one took its number and a stale
+  `memory:N` or `fact:N` named it. A remove now keeps the highest id handed out in `store_settings`
+  (`ids.memories.highest`, `ids.facts.highest`), and new memories and facts, restored project facts included, take
+  the next id above it. No schema change.
+
+## 0.505.0 — 31.10.2026
+
+### Added
+
+- **`appDoctorCommand` takes `--online`, and a messenger's and an app's `doctor` now say one thing.** Both use the
+  summary "the state this installation is in, without connecting unless --online" and the option text "also connect
+  once and read the account; sends nothing" (one constant, `doctor-text.ts`). Without the flag the answer is as before
+  and nothing is called. With it, `online` joins the answer: `{ ok, duration_ms, version: { current, latest, newer } }`,
+  plus the fields of the new `AppDoctorOptions.online(env)` hook, where a CLI checks its own service (a login, a folder
+  read); a throw there is `ok: false` with `error_code` and `error`, never a failed command. `version` comes from npm
+  (the `latestVersion` that `upgrade` uses) when the CLI passes the new `AppDoctorOptions.packageName`; without it that
+  check is left out. The command's effects are now `REMOTE_READ`, as the messenger's.
+
 ## 0.504.0 — 31.10.2026
 
 ### Added

package/dist/cli/app-doctor-command.d.ts

@@ -9,6 +9,15 @@ export interface AppDoctorOptions {
     oauthClient?: (env: NodeJS.ProcessEnv) => {
         client: "wirecat" | "own";
     };
+    /** The npm package, so `--online` can say whether a newer version is published; without it that check is left out. */
+    packageName?: string;
+    /**
+     * What `--online` checks of the service: a login, a folder read. Its fields join the `online` answer, and `ok: false`
+     * (or a throw, reported as `error_code` and `error`) makes the whole check fail without failing the command.
+     */
+    online?: (env: NodeJS.ProcessEnv) => Promise<{
+        ok: boolean;
+    } & Record<string, unknown>>;
 }
 /**
  * Run when everything else is broken, so a problem in the config file or the OAuth setting is a field, never a throw.
@@ -103,5 +112,9 @@ export declare const appDoctorReport: (app: AppIdentity, configuration: Configur
     };
     fixes: string[] | never[];
 };
-/** `doctor` for a CLI that is not a messenger: gcal, drive, zoom. A messenger's is `doctorCommand(messenger)`. */
+/**
+ * `doctor` for a CLI that is not a messenger: gcal, drive, zoom. A messenger's is `doctorCommand(messenger)`, and both
+ * say the same thing: the state from disk, and only with `--online` a call out. Here that call is `options.online`
+ * and, with `options.packageName`, whether npm has a newer version.
+ */
 export declare const appDoctorCommand: (app: AppIdentity, configuration: Configuration, options?: AppDoctorOptions) => Command;

package/dist/cli/app-doctor-command.js

@@ -2,15 +2,48 @@ import { existsSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { CliError, configFilePath, resolvePaths } from "@wirecat/cli-base";
 import { annotate } from "@wirecat/cli-base/commands";
+import { isNewer, latestVersion } from "@wirecat/cli-base/update";
 import { Command } from "commander";
 import { envName, readOnlyLockOf } from "../ports/app.js";
 import { storePath } from "../store/path.js";
 import { environmentOf, outputFor } from "./context.js";
-import { LOCAL_READ } from "./effects.js";
+import { DOCTOR_ONLINE_OPTION, DOCTOR_SUMMARY } from "./doctor-text.js";
+import { REMOTE_READ } from "./effects.js";
 import { itemOf, renderAnswer } from "./envelope.js";
 import { privateFiles } from "./messenger/private-files.js";
 import { unknownPermissionIssues } from "./permission-keys.js";
 import { profileFree } from "./profile.js";
+const onlineCheck = async (app, env, fetcher, options) => {
+    const started = Date.now();
+    let service = { ok: true };
+    if (options.online)
+        try {
+            service = await options.online(env);
+        }
+        catch (error) {
+            service = {
+                ok: false,
+                error_code: error instanceof CliError ? error.code : "generic_failure",
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    const latest = options.packageName
+        ? await latestVersion(options.packageName, fetcher, { timeoutMs: 3000 })
+        : undefined;
+    return {
+        ...service,
+        duration_ms: Date.now() - started,
+        ...(options.packageName
+            ? {
+                version: {
+                    current: app.version,
+                    latest: latest ?? null,
+                    newer: latest !== undefined && isNewer(latest, app.version),
+                },
+            }
+            : {}),
+    };
+};
 const failure = (error) => {
     if (error instanceof CliError)
         return error.message;
@@ -85,17 +118,26 @@ export const appDoctorReport = (app, configuration, env, command, options = {})
         };
     }
 };
-/** `doctor` for a CLI that is not a messenger: gcal, drive, zoom. A messenger's is `doctorCommand(messenger)`. */
+/**
+ * `doctor` for a CLI that is not a messenger: gcal, drive, zoom. A messenger's is `doctorCommand(messenger)`, and both
+ * say the same thing: the state from disk, and only with `--online` a call out. Here that call is `options.online`
+ * and, with `options.packageName`, whether npm has a newer version.
+ */
 export const appDoctorCommand = (app, configuration, options = {}) => profileFree(annotate(new Command("doctor"), {
-    effects: LOCAL_READ,
+    effects: REMOTE_READ,
     examples: [`${app.command} doctor`, `${app.command} doctor --json`],
 }))
-    .description("check this installation: version, paths, config, profiles, store; no keyring or network")
-    .action(function () {
-    const env = environmentOf(this).env ?? process.env;
+    .description(DOCTOR_SUMMARY)
+    .option("--online", DOCTOR_ONLINE_OPTION)
+    .action(async function () {
+    const environment = environmentOf(this);
+    const env = environment.env ?? process.env;
     const { report, fixes } = appDoctorReport(app, configuration, env, this, options);
+    const online = this.opts().online
+        ? await onlineCheck(app, env, environment.fetch ?? fetch, options)
+        : undefined;
     const output = outputFor(this);
-    renderAnswer(this, output, itemOf(report));
+    renderAnswer(this, output, itemOf(online ? { ...report, online } : report));
     for (const fix of fixes)
         output.renderer.note(`to make it private: ${fix}`);
 });

package/dist/cli/doctor-text.d.ts

@@ -0,0 +1,2 @@
+export declare const DOCTOR_SUMMARY = "the state this installation is in, without connecting unless --online";
+export declare const DOCTOR_ONLINE_OPTION = "also connect once and read the account; sends nothing";

package/dist/cli/doctor-text.js

@@ -0,0 +1,2 @@
+export const DOCTOR_SUMMARY = "the state this installation is in, without connecting unless --online";
+export const DOCTOR_ONLINE_OPTION = "also connect once and read the account; sends nothing";

package/dist/cli/messenger/doctor-command.js

@@ -10,6 +10,7 @@ import { FloodMemory, floodPathFor } from "../../sends/flood.js";
 import { SendJournal, sendsPathFor } from "../../sends/journal.js";
 import { storePath } from "../../store/path.js";
 import { baseContext, environmentOf, outputFor } from "../context.js";
+import { DOCTOR_ONLINE_OPTION, DOCTOR_SUMMARY } from "../doctor-text.js";
 import { destroysNothing, LOCAL_WRITE, NO_EFFECTS, REMOTE_READ } from "../effects.js";
 import { itemOf, renderAnswer, snakeKeys } from "../envelope.js";
 import { unknownPermissionIssues } from "../permission-keys.js";
@@ -28,8 +29,8 @@ import { storeSummary } from "./store-maintenance-command.js";
  */
 export const doctorCommand = (messenger) => {
     const command = annotate(new Command("doctor"), { effects: REMOTE_READ })
-        .description("the state this installation is in, without connecting unless --online")
-        .option("--online", "also connect once and read the account; sends nothing")
+        .description(DOCTOR_SUMMARY)
+        .option("--online", DOCTOR_ONLINE_OPTION)
         .action(async function () {
         const { online } = this.opts();
         const { report, renderer, fixes = [] } = await diagnose(this, messenger);

package/dist/mail/sync.js

@@ -1,3 +1,4 @@
+import { setTimeout } from "node:timers/promises";
 import { CliError } from "@wirecat/cli-base";
 import { importEngine } from "../attachments/extract.js";
 import { addressOf, FETCH_CHUNK, MAX_ATTACHMENT_CHARS, readAttachments } from "./attachments.js";
@@ -18,6 +19,8 @@ const DAY = 86_400_000;
 const EDGE = 2 * DAY;
 /** How often a long run saves where it is, so a kill costs at most this much listing again. */
 const SAVE_EVERY_MS = 30_000;
+const SLICE_SIZE = 50;
+const SLICE_PAUSE_MS = 100;
 /** About a minute of listing; the 30-day window of the timer stays below it. */
 const RESUME_FROM = 5_000;
 export { MAX_ATTACHMENT_CHARS } from "./attachments.js";
@@ -488,7 +491,8 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         inFolder.set(item.messageId, [...(inFolder.get(item.messageId) ?? []), Number(item.uid)]);
         uidsOf.set(folder, inFolder);
     }
-    const saveProgress = async () => {
+    const progressStates = () => {
+        const states = [];
         for (const [folder, searched] of searches) {
             if (searched.validity === undefined)
                 continue;
@@ -534,8 +538,13 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                 last: { ...thisRun, done: searched.uids.filter((uid) => within(merged, Number(uid))).length },
                 updatedAt: new Date(now()).toISOString(),
             };
-            await store.sync.setSyncState(key, resumeKey(folder), JSON.stringify(point));
+            states.push({ accountId, name: resumeKey(folder), value: JSON.stringify(point) });
         }
+        return states;
+    };
+    const saveProgress = async () => {
+        for (const { name, value } of progressStates())
+            await store.sync.setSyncState(key, name, value);
     };
     // Mail already stored is located before its UID is marked done, so a done UID always has its place.
     for (const folder of locatable) {
@@ -543,8 +552,39 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         await store.mail.setLocations(accountId, folder, searched.validity, uidsOf.get(folder) ?? new Map());
     }
     let savedAt = now();
+    // Fetched mail is saved in slices: one transaction holds the lock for a slice and carries the resume point, and the
+    // pause between slices lets another process take the lock (`await` over synchronous SQLite never yields).
+    const slice = [];
+    const flush = async (pause = false) => {
+        savedAt = now();
+        const batch = slice.splice(0);
+        if (batch.length === 0)
+            return saveProgress();
+        const ids = batch.flatMap(({ input }) => input.emails.map(({ externalId }) => externalId));
+        for (const id of ids)
+            settled.add(id);
+        try {
+            await store.mail.saveThreads(batch.map(({ input }) => input), progressStates());
+        }
+        catch (error) {
+            for (const id of ids)
+                settled.delete(id);
+            throw error;
+        }
+        for (const { done } of batch)
+            await done();
+        if (pause)
+            await setTimeout(SLICE_PAUSE_MS);
+    };
+    const enqueue = async (input, done) => {
+        slice.push({ input, done });
+        if (slice.length >= SLICE_SIZE)
+            await flush(true);
+    };
     const saveNow = async (force = false) => {
-        if (!force && now() - savedAt < SAVE_EVERY_MS)
+        if (force)
+            return flush();
+        if (now() - savedAt < SAVE_EVERY_MS)
             return;
         savedAt = now();
         await saveProgress();
@@ -560,7 +600,7 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
     for (const { item, rule } of toRecord) {
         await stopIfAborted();
         const headers = item.headers;
-        await store.mail.saveThread({
+        const input = {
             accountId,
             externalId: item.threadId,
             now: startedAt,
@@ -588,10 +628,11 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                     },
                 },
             ],
+        };
+        await enqueue(input, async () => {
+            envelopes++;
+            await tagStored(item);
         });
-        envelopes++;
-        settled.add(item.messageId);
-        await tagStored(item);
         await saveNow();
     }
     const fetchChunk = plan === undefined ? FETCH_CHUNK : (source.fetchBatch ?? FETCH_SLICE);
@@ -644,7 +685,7 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
             plan?.unreadable.delete(item.messageId);
             const header = mailHeadersOf(raws[index]);
             const files = await readAttachments(mail, item.messageId, { loadEngine, maxAttachmentChars, at: startedAt }, attachments);
-            await store.mail.saveThread({
+            const input = {
                 accountId,
                 externalId: item.threadId,
                 now: startedAt,
@@ -670,13 +711,14 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                         attachments: files,
                     },
                 ],
+            };
+            await enqueue(input, async () => {
+                saved++;
+                await tagStored(item);
+                const boxes = upgraded.get(item.messageId);
+                if (boxes !== undefined)
+                    await store.mail.setMailboxes(accountId, boxes.map(envelopeBoxOf), new Map([[item.messageId, []]]), startedAt);
             });
-            saved++;
-            settled.add(item.messageId);
-            await tagStored(item);
-            const boxes = upgraded.get(item.messageId);
-            if (boxes !== undefined)
-                await store.mail.setMailboxes(accountId, boxes.map(envelopeBoxOf), new Map([[item.messageId, []]]), startedAt);
         }
         await saveNow();
     }

package/dist/search/lucene/meaning-parts.d.ts

@@ -0,0 +1,14 @@
+import type { QueryNode } from "@wirecat/cli-base/query";
+type Predicate = Extract<QueryNode, {
+    kind: "predicate";
+}>;
+/**
+ * What a hit found by meaning must still match: the query with each word it asks for taken as matched, since the
+ * meaning half stands in for those. Words it excludes, and every other field, stay. `everything` is a predicate every
+ * record of the corpus matches, as a wanted word's stand-in: an empty clause list would match nothing. `undefined`
+ * when nothing but wanted words is left, so any hit found by meaning fits.
+ */
+export declare const fieldsOfQuery: (root: QueryNode, everything: Omit<Predicate, "span">) => QueryNode | undefined;
+/** The words and phrases the query asks for, for the meaning half to embed; none it excludes, no field. */
+export declare const meaningWords: (root: QueryNode) => string;
+export {};

package/dist/search/lucene/meaning-parts.js

@@ -0,0 +1,38 @@
+const WORD_FIELDS = new Set(["text", "exact"]);
+/**
+ * What a hit found by meaning must still match: the query with each word it asks for taken as matched, since the
+ * meaning half stands in for those. Words it excludes, and every other field, stay. `everything` is a predicate every
+ * record of the corpus matches, as a wanted word's stand-in: an empty clause list would match nothing. `undefined`
+ * when nothing but wanted words is left, so any hit found by meaning fits.
+ */
+export const fieldsOfQuery = (root, everything) => {
+    let narrows = false;
+    const kept = (node) => {
+        if (node.kind === "boolean")
+            return {
+                ...node,
+                clauses: node.clauses.map(({ occur, node }) => {
+                    if (occur !== "must_not")
+                        return { occur, node: kept(node) };
+                    narrows = true;
+                    return { occur, node };
+                }),
+            };
+        if (!WORD_FIELDS.has(node.field)) {
+            narrows = true;
+            return node;
+        }
+        return { ...everything, span: node.span };
+    };
+    const fields = kept(root);
+    return narrows ? fields : undefined;
+};
+/** The words and phrases the query asks for, for the meaning half to embed; none it excludes, no field. */
+export const meaningWords = (root) => {
+    const wanted = (node) => node.kind === "boolean"
+        ? node.clauses.flatMap(({ occur, node }) => (occur === "must_not" ? [] : wanted(node)))
+        : WORD_FIELDS.has(node.field) && (node.operator === "term" || node.operator === "phrase")
+            ? [node.value]
+            : [];
+    return wanted(root).join(" ").trim();
+};

package/dist/services/messages-discovery.d.ts

@@ -48,13 +48,28 @@ export interface MeaningSkipped {
  * The messages of these accounts nearest in meaning to the question, best first; or why there are none. `embedders`
  * is the search's own, when another part of it loads the same model.
  */
-export type MeaningLane = (question: string, accounts: AccountKey[], embedders?: Pick<WarmEmbedders, "get">) => Promise<{
+export type MeaningLane = (question: string, accounts: AccountKey[], embedders?: Pick<WarmEmbedders, "get">, 
+/**
+ * The store keys of a filter's own messages, to rank all of them instead of the nearest overall; `undefined` when
+ * the filter is too large. Asked only once meaning can run, so a search without the model never lists the filter.
+ */
+within?: () => Promise<number[] | undefined>) => Promise<{
     model: string;
     ranked: MeaningRanked;
     partly?: MeaningSkipped;
 } | {
     skipped: MeaningSkipped;
 }>;
+/**
+ * The most messages a filter may hold for meaning to rank every one of them that has a vector; above it, meaning
+ * takes the nearest overall and keeps those inside the filter, so a narrow filter loses some. Listing the filter costs
+ * ~2 µs a message and scoring ~22 µs a vector: on the owner's store (205k messages, 4% with vectors) a 30-day filter
+ * of 91k messages took ~0.2 s in all (cli-testing performance/search/messages-meaning-filters). Most messages have no
+ * vector, so the limit counts messages, not vectors; a store where all 100k have one would spend ~2 s.
+ */
+export declare const RANK_WITHIN_MESSAGES = 100000;
+/** What a hit not found by the query's words must match: as a search request's text or AST. */
+type Scope = Pick<SearchQuery, "text" | "ast">;
 /**
  * The query discovery searches. A question's own "?" would parse as a one-character wildcard and send it to strict
  * search (BUG-395); the colon of a time or ratio ("22:00", "1:1") would parse as a field and fail the whole search,
@@ -63,7 +78,7 @@ export type MeaningLane = (question: string, accounts: AccountKey[], embedders?:
 export declare const discoveryRequest: (asked: SearchQuery) => SearchQuery;
 export declare const searchDiscovery: (store: MessageStore, account: AccountKey, asked: SearchQuery, messenger?: Partial<Pick<MessengerTraits, "savedChatId" | "app" | "history" | "provider">>, meaning?: MeaningLane, rerank?: RerankLane) => Promise<SearchFound>;
 /** The nearest messages the question's filters allow, as hits, nearest first. */
-export declare const nearEligible: (store: MessageStore, account: AccountKey, request: SearchQuery, messenger: Partial<Pick<MessengerTraits, "savedChatId" | "app" | "history" | "provider">>, ranked: MeaningRanked, scope: string) => Promise<(FoundMessage & {
+export declare const nearEligible: (store: MessageStore, account: AccountKey, request: SearchQuery, messenger: Partial<Pick<MessengerTraits, "savedChatId" | "app" | "history" | "provider">>, ranked: MeaningRanked, scope: Scope) => Promise<(FoundMessage & {
     cosine: number;
 })[]>;
 /** What the reranker reads of a hit, as the benchmark gave it: its reply parent first, each line headed by where, who and when. */

package/dist/services/messages-discovery.js

@@ -1,7 +1,9 @@
 import { CliError } from "@wirecat/cli-base";
-import { implicitTextTerms, walkQuery } from "@wirecat/cli-base/query";
+import { implicitTextTerms, parseLucene, walkQuery } from "@wirecat/cli-base/query";
 import { parseLocator } from "../domain/locator.js";
-import { exhausted, QUERY_LIMITS } from "../search/lucene/types.js";
+import { fieldsOfQuery, meaningWords } from "../search/lucene/meaning-parts.js";
+import { validateFields } from "../search/lucene/registry.js";
+import { exhausted, MESSAGE_PARSING, QUERY_LIMITS } from "../search/lucene/types.js";
 import { planQuestion, words } from "../search/question-plan.js";
 import { searchCombined } from "./messages-combined.js";
 import { prepareLucene, searchLucene } from "./messages-search.js";
@@ -28,6 +30,25 @@ const FOLLOW_UPS_PER_HIT = 2;
 const RARE_AT = 3;
 /** Later messages read per hit before the search's filters; the latest that pass are kept. */
 const FOLLOW_UP_CANDIDATES = 20;
+/**
+ * The most messages a filter may hold for meaning to rank every one of them that has a vector; above it, meaning
+ * takes the nearest overall and keeps those inside the filter, so a narrow filter loses some. Listing the filter costs
+ * ~2 µs a message and scoring ~22 µs a vector: on the owner's store (205k messages, 4% with vectors) a 30-day filter
+ * of 91k messages took ~0.2 s in all (cli-testing performance/search/messages-meaning-filters). Most messages have no
+ * vector, so the limit counts messages, not vectors; a store where all 100k have one would spend ~2 s.
+ */
+export const RANK_WITHIN_MESSAGES = 100_000;
+/** Every message matches it: a wanted word's stand-in, and what a search with no text runs. */
+const EVERY_MESSAGE = {
+    kind: "predicate",
+    field: "date",
+    operator: "range",
+    value: "*",
+    upper: "*",
+    lower_inclusive: true,
+    upper_inclusive: true,
+};
+const scopeOf = (root) => root === undefined ? {} : { ast: { version: 1, language: "lucene-v1", root } };
 /**
  * The query discovery searches. A question's own "?" would parse as a one-character wildcard and send it to strict
  * search (BUG-395); the colon of a time or ratio ("22:00", "1:1") would parse as a field and fail the whole search,
@@ -46,20 +67,35 @@ export const searchDiscovery = async (store, account, asked, messenger = {}, mea
     const original = await prepareLucene(store, account, request, messenger);
     const text = request.text ?? "";
     const plan = planQuestion(text, request.ast !== undefined || !!request.exact || !!request.newest);
-    if (request.ast !== undefined || request.exact || request.newest || implicitTextTerms(text).size === 0)
+    if (request.ast !== undefined || request.exact || request.newest)
         return searchLucene(store, account, request, messenger);
     const implicit = implicitTextTerms(text);
-    if (!plan.changed &&
-        walkQuery(original.execution.root).some((node) => (node.field === "text" || node.field === "exact") &&
-            (node.operator !== "term" || node.field === "exact" || !implicit.has(node.span.start))))
+    // Explicit syntax (-, OR, NOT, groups, phrases, exact:, wildcards) keeps the whole query for the words; meaning
+    // reads only its wanted words, and its hits must still match every field and every excluded word.
+    const explicit = implicit.size === 0 ||
+        (!plan.changed &&
+            walkQuery(original.execution.root).some((node) => (node.field === "text" || node.field === "exact") &&
+                (node.operator !== "term" || node.field === "exact" || !implicit.has(node.span.start))));
+    const root = explicit
+        ? validateFields(parseLucene(text, { ...MESSAGE_PARSING, defaultField: "text" })).root
+        : undefined;
+    const question = root === undefined ? text : meaningWords(root);
+    if (!question)
         return searchLucene(store, account, request, messenger);
-    const terms = [...new Set(plan.terms.concat(plan.aliases.flatMap((alias) => words(alias.split(" -> ")[1] ?? ""))))];
+    const terms = root === undefined
+        ? [...new Set(plan.terms.concat(plan.aliases.flatMap((alias) => words(alias.split(" -> ")[1] ?? ""))))]
+        : [...new Set(words(question))];
     if (!terms.length)
         return searchLucene(store, account, request, messenger);
     if (terms.length > 64)
         exhausted("discovery terms");
+    const narrowing = root === undefined ? undefined : fieldsOfQuery(root, EVERY_MESSAGE);
+    const scope = root === undefined ? (plan.scope ? { text: plan.scope } : {}) : scopeOf(narrowing);
     // Before the clock starts: loading the model alone takes about a second of the word search's budget.
-    const lane = meaning ? await meaning(text, original.scopeAccounts) : undefined;
+    const within = scope.text !== undefined || scope.ast !== undefined
+        ? () => filterPks(store, account, { ...request, text: undefined, ast: undefined, ...scope }, messenger)
+        : undefined;
+    const lane = meaning ? await meaning(question, original.scopeAccounts, undefined, within) : undefined;
     let started = performance.now();
     const check = () => {
         if (request.signal?.aborted)
@@ -75,7 +111,7 @@ export const searchDiscovery = async (store, account, asked, messenger = {}, mea
         union.set(m.locator, { ...previous, ...m, fused: (previous?.fused ?? 0) + 1 / (60 + rank + 1) });
     });
     // Natural questions use one ranked partial lookup; repeated all-word lookups cannot rescue their wording gaps.
-    for (const query of plan.changed ? [] : plan.queries) {
+    for (const query of plan.changed || root !== undefined ? [] : plan.queries) {
         check();
         const found = await searchCombined(store, account, { ...request, text: query, limit: 300 }, messenger, {
             candidateDepth: 300,
@@ -89,7 +125,7 @@ export const searchDiscovery = async (store, account, asked, messenger = {}, mea
         truncated ||= found.hasMore || found.query.combined.truncated;
     }
     check();
-    const relaxed = `(${terms.join(" OR ")})${plan.scope ? ` ${plan.scope}` : ""}`;
+    const relaxed = root === undefined ? `(${terms.join(" OR ")})${plan.scope ? ` ${plan.scope}` : ""}` : text;
     const partialRequest = { ...request, text: relaxed, ast: undefined, limit: 300, context: 0, thread: undefined };
     const partial = await searchLucene(store, account, partialRequest, messenger);
     add(partial.items);
@@ -116,8 +152,9 @@ export const searchDiscovery = async (store, account, asked, messenger = {}, mea
                 request.only.some((key) => key.provider === m.provider && key.account === m.account && key.chatId === m.chatId && key.id === m.id));
             const eligible = await searchLucene(store, account, {
                 ...request,
-                text: plan.scope || undefined,
+                text: undefined,
                 ast: undefined,
+                ...scope,
                 limit: 500,
                 only,
                 context: 0,
@@ -178,7 +215,7 @@ export const searchDiscovery = async (store, account, asked, messenger = {}, mea
         Number(parentByLocator.has(b.locator)) - Number(parentByLocator.has(a.locator)));
     let meaningCandidates = 0;
     if (lane && "ranked" in lane) {
-        const near = await nearEligible(store, account, request, messenger, lane.ranked, plan.scope);
+        const near = await nearEligible(store, account, request, messenger, lane.ranked, scope);
         meaningCandidates = near.length;
         candidates = fusedWithMeaning(candidates, near, described);
     }
@@ -228,7 +265,7 @@ export const searchDiscovery = async (store, account, asked, messenger = {}, mea
         started += performance.now() - opening;
     }
     check();
-    const followUps = await followUpsOf(store, account, request, messenger, candidates, plan.scope);
+    const followUps = await followUpsOf(store, account, request, messenger, candidates, scope);
     const items = await Promise.all(candidates.slice(0, request.limit).map(async ({ fused: _fused, ...hit }) => {
         const later = followUps.get(hit.locator);
         const m = later?.length ? { ...hit, followUps: later } : hit;
@@ -254,14 +291,14 @@ export const searchDiscovery = async (store, account, asked, messenger = {}, mea
         query: {
             ...partial.query,
             discovery: {
-                method: "lexical-partial",
+                method: root === undefined ? "lexical-partial" : "lexical-strict",
                 candidateDepth: 300,
                 candidates: candidates.length,
                 repliesAdded: replyCount,
                 followUps: [...followUps.values()].reduce((sum, later) => sum + later.length, 0),
                 truncated,
                 terms,
-                queries: [...(plan.changed ? [] : plan.queries), relaxed],
+                queries: [...(plan.changed || root !== undefined ? [] : plan.queries), relaxed],
                 ...(lane && "ranked" in lane ? { meaning: { model: lane.model, candidates: meaningCandidates } } : {}),
                 ...(reranked ? { rerank: reranked } : {}),
             },
@@ -286,7 +323,7 @@ const followUpsOf = async (store, account, request, messenger, candidates, scope
     const out = new Map();
     if (!only.length)
         return out;
-    const eligible = await searchLucene(store, account, { ...request, text: scope || undefined, ast: undefined, limit: only.length, only, context: 0, thread: undefined }, messenger);
+    const eligible = await searchLucene(store, account, { ...request, text: undefined, ast: undefined, ...scope, limit: only.length, only, context: 0, thread: undefined }, messenger);
     const allowed = new Map(eligible.items.map((m) => [m.locator, m]));
     const shown = new Set();
     head.forEach((hit, i) => {
@@ -302,6 +339,15 @@ const followUpsOf = async (store, account, request, messenger, candidates, scope
     });
     return out;
 };
+/**
+ * The store keys of the messages a filter holds, when there are at most `RANK_WITHIN_MESSAGES`; `undefined` when more,
+ * or when only a full search could list them.
+ */
+const filterPks = async (store, account, request, messenger) => {
+    const { execution } = await prepareLucene(store, account, request, messenger);
+    const pks = await store.queries.matchingPks(execution, RANK_WITHIN_MESSAGES);
+    return pks === undefined || pks.length > RANK_WITHIN_MESSAGES ? undefined : pks;
+};
 /** The nearest messages the question's filters allow, as hits, nearest first. */
 export const nearEligible = async (store, account, request, messenger, ranked, scope) => {
     const keyOf = (one) => JSON.stringify([one.provider, one.account, one.chatId, one.id]);
@@ -311,7 +357,7 @@ export const nearEligible = async (store, account, request, messenger, ranked, s
         .map(({ provider, account, chatId, id }) => ({ provider, account, chatId, id }));
     if (!only.length)
         return [];
-    const eligible = await searchLucene(store, account, { ...request, text: scope || undefined, ast: undefined, limit: only.length, only, context: 0, thread: undefined }, messenger);
+    const eligible = await searchLucene(store, account, { ...request, text: undefined, ast: undefined, ...scope, limit: only.length, only, context: 0, thread: undefined }, messenger);
     const hits = new Map(eligible.items.map((hit) => {
         const { provider, account } = parseLocator(hit.locator);
         return [keyOf({ provider, account, chatId: hit.chatId, id: hit.id }), hit];

package/dist/services/messages-meaning.js

@@ -6,7 +6,7 @@ import { plainWords } from "./pictures.js";
  * Discovery's meaning half: the default local model over each message's own vector. It runs only when the model is
  * on disk and vectors exist — never a download, never a remote model — and any failure leaves the words alone.
  */
-export const meaningLane = (deps) => async (question, accounts, shared) => {
+export const meaningLane = (deps) => async (question, accounts, shared, within) => {
     const target = embeddingTarget(deps, DEFAULT_TEXT_MODEL);
     const skipped = (reason, step) => ({
         skipped: { reason, model: target.id, ...(step ? { step } : {}) },
@@ -24,14 +24,17 @@ export const meaningLane = (deps) => async (question, accounts, shared) => {
     try {
         const ranked = [];
         if (words) {
+            const pks = await within?.();
             const lookUp = async (embedder) => {
                 const [query] = await embedder.embed([words], "query");
                 for (const one of ready) {
-                    const near = await store.vectors.nearestMessages(one, {
-                        model: target.key,
-                        limit: MEANING_DEPTH,
-                        query: query,
-                    });
+                    const near = pks
+                        ? await store.vectors.scoreMessages(one, { model: target.key, query: query, pks })
+                        : await store.vectors.nearestMessages(one, {
+                            model: target.key,
+                            limit: MEANING_DEPTH,
+                            query: query,
+                        });
                     for (const { chatId, messageId, score } of near)
                         ranked.push({ ...one, chatId, id: messageId, score });
                 }

package/dist/services/messages-search.d.ts

@@ -8,7 +8,8 @@ import type { SearchRefreshed } from "./search-refresh.js";
 export interface QueryMetadata {
     language: "lucene-v1";
     discovery?: {
-        method: "lexical-partial";
+        /** `lexical-strict` when the query's own syntax (-, OR, NOT, groups…) chose the word hits, not a relaxed OR. */
+        method: "lexical-partial" | "lexical-strict";
         candidateDepth: number;
         candidates: number;
         repliesAdded: number;

package/dist/services/notes-search.d.ts

@@ -169,14 +169,7 @@ export declare const noteSourceOfQuery: (query: string) => {
     source?: Note["source"];
     rest: string;
 };
-/**
- * What a note found by meaning must still match: the query with each word it asks for taken as matched, since the
- * meaning half stands in for those. Words it excludes, and every other field, stay. `undefined` when nothing but
- * wanted words is left, so any note found by meaning fits.
- */
 export declare const fieldsOfQuery: (root: QueryNode) => QueryNode | undefined;
-/** The words and phrases the query asks for, for the meaning half to embed; none it excludes, no field. */
-export declare const meaningWords: (root: QueryNode) => string;
 /**
  * The most notes a filter may hold for meaning to rank all of them, each by its best chunk. Above it, meaning ranks
  * the nearest notes overall and keeps those inside the filter, so a narrow filter over a large store loses some. On

package/dist/services/notes-search.js

@@ -6,6 +6,7 @@ import { defaultThreads, isTextModelInstalled, openEmbedder, textModelsDirectory
 import { DEFAULT_TEXT_MODEL, meaningFloor, textModel, vectorModelKey } from "../embeddings/models.js";
 import { DEFAULT_IMAGE_MODEL, imageModel } from "../images/picture/models.js";
 import { entriesOfList, fitting } from "../search/account-list.js";
+import { fieldsOfQuery as fieldsOf, meaningWords } from "../search/lucene/meaning-parts.js";
 import { validateFields } from "../search/lucene/registry.js";
 import { hasStems } from "../search/lucene/resolved.js";
 import { SEARCH_QUERY } from "../search/lucene/types.js";
@@ -234,44 +235,9 @@ export const noteSourceOfQuery = (query) => {
     }
     return { source, rest };
 };
-const WORD_FIELDS = new Set(["text", "exact"]);
-/**
- * What a note found by meaning must still match: the query with each word it asks for taken as matched, since the
- * meaning half stands in for those. Words it excludes, and every other field, stay. `undefined` when nothing but
- * wanted words is left, so any note found by meaning fits.
- */
-export const fieldsOfQuery = (root) => {
-    let narrows = false;
-    const kept = (node) => {
-        if (node.kind === "boolean")
-            return {
-                ...node,
-                clauses: node.clauses.map(({ occur, node }) => {
-                    if (occur !== "must_not")
-                        return { occur, node: kept(node) };
-                    narrows = true;
-                    return { occur, node };
-                }),
-            };
-        if (!WORD_FIELDS.has(node.field)) {
-            narrows = true;
-            return node;
-        }
-        // `in:notes` holds for every note and document; an empty clause list would match nothing.
-        return { kind: "predicate", field: "in", operator: "term", value: "notes", span: node.span };
-    };
-    const fields = kept(root);
-    return narrows ? fields : undefined;
-};
-/** The words and phrases the query asks for, for the meaning half to embed; none it excludes, no field. */
-export const meaningWords = (root) => {
-    const wanted = (node) => node.kind === "boolean"
-        ? node.clauses.flatMap(({ occur, node }) => (occur === "must_not" ? [] : wanted(node)))
-        : WORD_FIELDS.has(node.field) && (node.operator === "term" || node.operator === "phrase")
-            ? [node.value]
-            : [];
-    return wanted(root).join(" ").trim();
-};
+/** `in:notes` holds for every note and document. */
+const EVERY_NOTE = { kind: "predicate", field: "in", operator: "term", value: "notes" };
+export const fieldsOfQuery = (root) => fieldsOf(root, EVERY_NOTE);
 const allOf = (nodes) => nodes.length <= 1
     ? nodes[0]
     : {
@@ -286,7 +252,7 @@ const allOf = (nodes) => nodes.length <= 1
 const matching = async (store, refs, root, scope) => {
     if (refs.length === 0)
         return new Map();
-    const resolved = await resolvedQuery(store, root ?? { kind: "predicate", field: "in", operator: "term", value: "notes", span: { start: 0, end: 0 } }, scope.timezone);
+    const resolved = await resolvedQuery(store, root ?? { ...EVERY_NOTE, span: { start: 0, end: 0 } }, scope.timezone);
     const found = await store.notes.search({
         root: resolved.root,
         refs,

package/dist/services/search-all.js

@@ -123,7 +123,7 @@ export const searchAll = async (store, account, request, messenger = {}, searchM
             return byWords;
         }
         messagesBy = { by: "words and meaning", ...(lane.partly ? { meaningSkipped: lane.partly } : {}) };
-        const near = await nearEligible(store, account, query, messenger, lane.ranked, filters ?? "");
+        const near = await nearEligible(store, account, query, messenger, lane.ranked, filters ? { text: filters } : {});
         // One list per resource, as notes give theirs, so messages get no second vote against mail.
         const fused = new Map();
         byWords.forEach((item, rank) => {

package/dist/store/index.d.ts

@@ -51,6 +51,7 @@ export type { PrivateContact, PrivateContactNote } from "./sqlite/private-people
 export type { ProposalInput, ProposedAction, ProposedActionsStore } from "./sqlite/proposed-actions.js";
 export { PROPOSAL_STATUSES } from "./sqlite/proposed-actions.js";
 export { fillSearchIndex, resetSearchIndex, type SearchIndexFill, type SearchIndexState, searchIndexState, } from "./sqlite/search-index.js";
+export type { SyncStateSave } from "./sqlite/sync.js";
 export type { TagCreationOptions, TagCreationReview } from "./sqlite/tag-creation.js";
 export type { TagKind, TagRow } from "./sqlite/tags.js";
 export type { SimilarSubject, SimilarTask, TaskDraft } from "./sqlite/task-candidates.js";

package/dist/store/queries.d.ts

@@ -15,6 +15,8 @@ export interface QueriesStore {
     countQuery(execution: QueryExecution, by: QueryGrouping): Promise<QueryGroup[]>;
     /** How many messages match, counted no further than `cap + 1`; `undefined` when only a full search could tell. */
     countMatches(execution: QueryExecution, cap: number): Promise<number | undefined>;
+    /** The keys of the messages that match, no more than `cap + 1`; `undefined` when only a full search could tell. */
+    matchingPks(execution: QueryExecution, cap: number): Promise<number[] | undefined>;
     conversationEligibility(execution: QueryExecution): Promise<ConversationEligibility>;
     rankQuery(execution: QueryExecution, request: RankingRequest): Promise<RankedStoreFound>;
     rankingEvidence(execution: QueryExecution, request: RankingEvidenceRequest): Promise<RankedEvidence>;

package/dist/store/queries.js

@@ -14,6 +14,7 @@ export const queriesStoreOver = (context, stemmerFor) => {
         matchQuery: async (execution) => lucene.matchQuery(context, mailIndexed(execution)),
         countQuery: async (execution, by) => lucene.countQuery(context, mailIndexed(execution), by),
         countMatches: async (execution, cap) => lucene.countMatches(context, mailIndexed(execution), cap),
+        matchingPks: async (execution, cap) => lucene.matchingPks(context, mailIndexed(execution), cap),
         conversationEligibility: async (execution) => conversationEligibility(context, execution),
         rankQuery: async (execution, request) => rankQuery(context, execution, request),
         rankingEvidence: async (execution, request) => rankingEvidence(context, execution, request),

package/dist/store/sqlite/emails.d.ts

@@ -7,6 +7,7 @@ import { type EmailLocationInput, type FolderEmail } from "./email-locations.js"
 import { type MailRulesStore } from "./mail-rules.js";
 import { type AttachmentInput } from "./meetings.js";
 import type { StoreContext } from "./open.js";
+import { type SyncStateSave } from "./sync.js";
 export type { EmailLocationInput, FolderEmail };
 export interface EmailAddress {
     address: string;
@@ -203,6 +204,8 @@ export interface SenderScore {
 /** Mail in tables of its own: threads, emails, recipients and mailboxes, keyed by account and Message-ID. */
 export interface MailStore {
     saveThread(input: ThreadSave): Promise<ThreadDetails>;
+    /** Several threads and sync state in one transaction: a slice of mail commits with its resume point, or not at all. */
+    saveThreads(inputs: ThreadSave[], states?: SyncStateSave[]): Promise<ThreadDetails[]>;
     /** Newest activity first. */
     threads(filter?: MailFilter): Promise<EmailThread[]>;
     thread(id: number): Promise<ThreadDetails | null>;
@@ -280,4 +283,4 @@ export declare const emailHitsByPk: (database: CacheDatabase, pks: number[]) =>
  * Indexes the queued emails as the notes are indexed, words and stems of the subject, then the body; then cuts
  * the threads they belong to into pieces for search by meaning.
  */
-export declare const mailStoreOver: ({ database, orm }: Pick<StoreContext, "database" | "orm">, stemmerFor?: (stemmers: Stemmers) => Stemmer) => MailStore;
+export declare const mailStoreOver: ({ database, orm, now }: Pick<StoreContext, "database" | "orm" | "now">, stemmerFor?: (stemmers: Stemmers) => Stemmer) => MailStore;

package/dist/store/sqlite/emails.js

@@ -10,6 +10,7 @@ import { ensurePerson } from "./identities.js";
 import { mailRulesOver } from "./mail-rules.js";
 import { attachmentOf, saveAttachment, wordsQuery } from "./meetings.js";
 import { stemmerCache } from "./stems.js";
+import { writeState } from "./sync.js";
 import { write } from "./transaction.js";
 import { dropUnusedVectors } from "./vector-index.js";
 import { dot } from "./vectors.js";
@@ -415,12 +416,21 @@ export const emailHitsByPk = (database, pks) => {
  * Indexes the queued emails as the notes are indexed, words and stems of the subject, then the body; then cuts
  * the threads they belong to into pieces for search by meaning.
  */
-export const mailStoreOver = ({ database, orm }, stemmerFor = stemmerCache()) => ({
+export const mailStoreOver = ({ database, orm, now }, stemmerFor = stemmerCache()) => ({
     rules: mailRulesOver(database),
     async saveThread(input) {
         const id = write(database, () => saveThread(database, orm, input));
         return threadDetails(database, id);
     },
+    async saveThreads(inputs, states = []) {
+        const ids = write(database, () => {
+            const saved = inputs.map((input) => saveThread(database, orm, input));
+            for (const { accountId, name, value } of states)
+                writeState({ orm, now }, accountId, name, value);
+            return saved;
+        });
+        return ids.map((id) => threadDetails(database, id));
+    },
     async threads(filter = {}) {
         const [limit, offset] = page(filter);
         return database

package/dist/store/sqlite/facts.js

@@ -3,7 +3,7 @@ import { FACT_TYPES, MEMORY_SCOPES } from "../../domain/knowledge.js";
 import { calendarMoment } from "../../domain/moment.js";
 import { evidenceSourcesOf } from "./evidence-sources.js";
 import { linkKind } from "./link-kinds.js";
-import { authorActor, authorOf, dropLinksOf, erase, evidenceOf, linkEvidence, subjectOf, } from "./memories.js";
+import { authorActor, authorOf, dropLinksOf, erase, evidenceOf, keepHighestId, linkEvidence, nextId, subjectOf, } from "./memories.js";
 import { requiredThing } from "./notes.js";
 import { write } from "./transaction.js";
 const NAMED = "SELECT f.*, b.name AS author_name FROM facts f LEFT JOIN bots b ON f.author_type = 'bot' AND b.id = f.author_id";
@@ -80,10 +80,10 @@ export const factOperationsOver = (context) => {
                 const { actor, model } = authorActor(context, input.by);
                 const at = now();
                 const made = Number(database
-                    .prepare("INSERT INTO facts (type, statement, status, project_id, subject_type, subject_id, occurred_at, supersedes_id, " +
+                    .prepare("INSERT INTO facts (id, type, statement, status, project_id, subject_type, subject_id, occurred_at, supersedes_id, " +
                     "author_type, author_id, model, scope, source, created_at, updated_at) " +
-                    "VALUES (?, ?, 'proposed', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id")
-                    .get(type, statement, project?.id ?? null, subject?.type ?? null, subject?.id ?? null, occurredAt, replaced === undefined ? null : Number(replaced.id), actor.type, actor.id, model, scope, owner ? "owner" : "agent", at, at)?.id);
+                    "VALUES (?, ?, ?, 'proposed', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id")
+                    .get(nextId(database, "facts"), type, statement, project?.id ?? null, subject?.type ?? null, subject?.id ?? null, occurredAt, replaced === undefined ? null : Number(replaced.id), actor.type, actor.id, model, scope, owner ? "owner" : "agent", at, at)?.id);
                 linkEvidence(context, { type: "fact", id: made }, input.evidence ?? [], owner ? "owner" : "agent");
                 if (input.createdFrom !== undefined) {
                     const from = requiredThing(database, input.createdFrom, "the source");
@@ -135,6 +135,9 @@ export const factOperationsOver = (context) => {
             const id = Number(row.id);
             const twin = row.source === "memory" && row.metadata != null ? migratedFrom(String(row.metadata)) : undefined;
             const links = erase(database, () => {
+                keepHighestId(database, "facts", now());
+                if (twin !== undefined)
+                    keepHighestId(database, "memories", now());
                 database.prepare("UPDATE facts SET supersedes_id = NULL WHERE supersedes_id = ?").run(id);
                 const count = dropLinksOf(database, { type: "fact", id });
                 database.prepare("DELETE FROM links WHERE from_type = 'fact' AND from_id = ?").run(id);

package/dist/store/sqlite/follow-ups.js

@@ -17,14 +17,14 @@ export const laterSharingRare = (context, wanted, { rareAt, limit }) => {
     if (!wanted.length || !stemmers || !stemsState(database)?.ready)
         return wanted.map(() => []);
     const stemmer = stemmerFor(stemmers);
-    // A temporary table: reading the index's own counts needs no migration, and it leaves with the connection.
-    database.exec("CREATE VIRTUAL TABLE IF NOT EXISTS temp.message_stem_counts USING fts5vocab(main, message_stems, col)");
-    const count = database.prepare("SELECT doc FROM temp.message_stem_counts WHERE term = ? AND col = 'stems'");
+    // Counted only up to one past `rareAt`, so a common word costs no more than a rare one. Not an fts5vocab table:
+    // creating one is a write, which a read-only connection refuses.
+    const count = database.prepare("SELECT count(*) AS doc FROM (SELECT rowid FROM message_stems WHERE message_stems MATCH ? LIMIT ?)");
     const counts = new Map();
     const countOf = (stem) => {
         let found = counts.get(stem);
         if (found === undefined) {
-            found = Number(count.get(stem)?.doc ?? 0);
+            found = Number(count.get(`stems : "${stem.replaceAll('"', '""')}"`, rareAt + 1)?.doc ?? 0);
             counts.set(stem, found);
         }
         return found;

package/dist/store/sqlite/lucene.d.ts

@@ -18,11 +18,13 @@ export declare const matchQuery: (context: StoreContext, execution: QueryExecuti
 export declare const countQuery: (context: StoreContext, execution: QueryExecution, by: QueryGrouping) => Promise<QueryGroup[]>;
 /** All matching message keys, without materializing message bodies or imposing a hit-page limit. */
 export declare const queryMessagePks: (context: StoreContext, execution: QueryExecution) => Promise<number[]>;
+/** How many messages match, counting no further than `cap + 1`; `undefined` as `matchingPks` says. */
+export declare const countMatches: (context: StoreContext, execution: QueryExecution, cap: number) => number | undefined;
 /**
- * How many messages match, counting no further than `cap + 1`: one SQL count, unranked. `undefined` when a leaf is
- * checked row by row (a preset, a pattern), where a count would cost as much as the search itself.
+ * The keys of the messages that match, no more than `cap + 1`: one SQL query, unranked. `undefined` when a leaf is
+ * checked row by row (a preset, a pattern), where it would cost as much as the search itself.
  */
-export declare const countMatches: (context: StoreContext, execution: QueryExecution, cap: number) => number | undefined;
+export declare const matchingPks: (context: StoreContext, execution: QueryExecution, cap: number) => number[] | undefined;
 interface QuerySelection {
     sql: string;
     params: SqlValue[];

package/dist/store/sqlite/lucene.js

@@ -48,22 +48,24 @@ export const matchQuery = (context, execution) => runQuery(context, execution);
 export const countQuery = (context, execution, by) => runQuery(context, execution, by);
 /** All matching message keys, without materializing message bodies or imposing a hit-page limit. */
 export const queryMessagePks = (context, execution) => runQuery(context, execution, "pks");
+/** How many messages match, counting no further than `cap + 1`; `undefined` as `matchingPks` says. */
+export const countMatches = (context, execution, cap) => matchingPks(context, execution, cap)?.length;
 /**
- * How many messages match, counting no further than `cap + 1`: one SQL count, unranked. `undefined` when a leaf is
- * checked row by row (a preset, a pattern), where a count would cost as much as the search itself.
+ * The keys of the messages that match, no more than `cap + 1`: one SQL query, unranked. `undefined` when a leaf is
+ * checked row by row (a preset, a pattern), where it would cost as much as the search itself.
  */
-export const countMatches = (context, execution, cap) => {
+export const matchingPks = (context, execution, cap) => {
     const { database, check, where, from, withRanking, ranked, leaves } = compileQuery(context, {
         ...execution,
         newest: true,
     });
     if (leaves.some(({ test }) => test))
         return undefined;
-    const row = database
-        .prepare(`SELECT count(*) AS n FROM (${withRanking}SELECT m.id ${from} WHERE ${where.sql} LIMIT ?)`)
-        .get(...ranked.params, ...where.params, cap + 1);
+    const rows = database
+        .prepare(`${withRanking}SELECT m.id ${from} WHERE ${where.sql} LIMIT ?`)
+        .all(...ranked.params, ...where.params, cap + 1);
     check();
-    return Number(row?.n ?? 0);
+    return rows.map(({ id }) => Number(id));
 };
 const compileQuery = (context, execution, boundedAttachments = false) => {
     const { database } = context;

package/dist/store/sqlite/memories.d.ts

@@ -100,6 +100,14 @@ export declare const evidenceOf: (context: StoreContext, from: {
 }) => string[];
 /** What a memory's or fact's subject is called, for a reader who should not have to look the reference up. */
 export declare const subjectOf: (database: StoreContext["database"], row: Row) => Pick<Memory, "subject" | "subjectName" | "subjectIsOwner">;
+type Numbered = "memories" | "facts";
+/**
+ * The id the next row takes: above every row there is and every one forgotten. SQLite hands out the highest id + 1,
+ * so without this a forgotten record's number would name the next one, and a stale `memory:N` act on it.
+ */
+export declare const nextId: (database: StoreContext["database"], table: Numbered) => number;
+/** Before a delete: remembers the highest id handed out, which the delete may be about to free. */
+export declare const keepHighestId: (database: StoreContext["database"], table: Numbered, at: number) => void;
 /** Counts every link from or to a row and drops the ones its delete trigger does not: those pointing at it. */
 export declare const dropLinksOf: (database: StoreContext["database"], thing: {
     type: string;

package/dist/store/sqlite/memories.js

@@ -64,6 +64,23 @@ export const subjectOf = (database, row) => {
         subjectIsOwner: Number(named?.owner) === 1,
     };
 };
+const highestKey = (table) => `ids.${table}.highest`;
+/**
+ * The id the next row takes: above every row there is and every one forgotten. SQLite hands out the highest id + 1,
+ * so without this a forgotten record's number would name the next one, and a stale `memory:N` act on it.
+ */
+export const nextId = (database, table) => {
+    const kept = database.prepare("SELECT value FROM store_settings WHERE key = ?").get(highestKey(table));
+    const highest = database.prepare(`SELECT max(id) AS id FROM ${table}`).get();
+    return Math.max(Number(kept?.value ?? 0), Number(highest?.id ?? 0)) + 1;
+};
+/** Before a delete: remembers the highest id handed out, which the delete may be about to free. */
+export const keepHighestId = (database, table, at) => {
+    const highest = nextId(database, table) - 1;
+    database
+        .prepare("INSERT OR REPLACE INTO store_settings (key, value, at) VALUES (?, ?, ?)")
+        .run(highestKey(table), String(highest), at);
+};
 /** Counts every link from or to a row and drops the ones its delete trigger does not: those pointing at it. */
 export const dropLinksOf = (database, thing) => {
     const count = Number(database
@@ -153,9 +170,9 @@ export const memoriesStoreOver = (context) => {
                 const replaced = input.supersedes === undefined ? undefined : Number(rowOf(input.supersedes).id);
                 const at = now();
                 const made = Number(database
-                    .prepare("INSERT INTO memories (kind, body, subject_type, subject_id, author_type, author_id, model, confidence, status, " +
-                    "last_verified_at, supersedes_id, scope, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id")
-                    .get(kind, body, subject?.type ?? null, subject?.id ?? null, actor.type, actor.id, model, input.confidence ?? null, status, status === "confirmed" ? at : null, replaced ?? null, scope, at, at)?.id);
+                    .prepare("INSERT INTO memories (id, kind, body, subject_type, subject_id, author_type, author_id, model, confidence, status, " +
+                    "last_verified_at, supersedes_id, scope, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id")
+                    .get(nextId(database, "memories"), kind, body, subject?.type ?? null, subject?.id ?? null, actor.type, actor.id, model, input.confidence ?? null, status, status === "confirmed" ? at : null, replaced ?? null, scope, at, at)?.id);
                 linkEvidence(context, { type: "memory", id: made }, input.evidence, input.author === "owner" ? "owner" : "agent");
                 if (replaced !== undefined)
                     database.prepare("UPDATE memories SET status = 'superseded', updated_at = ? WHERE id = ?").run(at, replaced);
@@ -178,6 +195,7 @@ export const memoriesStoreOver = (context) => {
         remove: async (reference) => {
             const id = Number(rowOf(reference).id);
             const links = erase(database, () => {
+                keepHighestId(database, "memories", now());
                 database.prepare("UPDATE memories SET supersedes_id = NULL WHERE supersedes_id = ?").run(id);
                 const count = dropLinksOf(database, { type: "memory", id });
                 database.prepare("DELETE FROM memories WHERE id = ?").run(id);

package/dist/store/sqlite/message-chunks.d.ts

@@ -39,6 +39,12 @@ type MessageFilter = {
     since?: number;
     before?: number;
 };
+/** Each of the account's messages among `pks` that has a vector of `model`, scored against `query`, best first. */
+export declare const scoreMessages: ({ orm }: StoreContext, accountPk: number, { model, query, pks }: {
+    model: string;
+    query: Float32Array;
+    pks: number[];
+}) => NearestMessage[];
 /** The account's live messages nearest to `query` by their vector of `model`, best first. */
 export declare const nearestMessages: (context: StoreContext, accountPk: number, { limit, query, ...filter }: MessageFilter & {
     limit: number;

package/dist/store/sqlite/message-chunks.js

@@ -59,6 +59,16 @@ const messageRows = ({ orm }, accountPk, { chatKey, model, since, before }, rows
         ${"ids" in rows
     ? sql `AND k.id IN (SELECT value FROM json_each(${JSON.stringify(rows.ids)})) ORDER BY k.id`
     : sql `AND k.id > ${rows.after} ORDER BY k.id LIMIT ${SCAN_PAGE}`}`);
+/** Each of the account's messages among `pks` that has a vector of `model`, scored against `query`, best first. */
+export const scoreMessages = ({ orm }, accountPk, { model, query, pks }) => orm
+    .all(sql `SELECT ch.external_id AS chatId, m.external_id AS messageId, v.vector
+        FROM chunks k CROSS JOIN messages m ON m.id = k.chunkable_id AND m.deleted_at IS NULL
+          JOIN chats ch ON ch.id = m.chat_id AND ch.searchable = 1
+          JOIN embeddings v ON v.model = ${model} AND v.content_hash = k.content_hash
+        WHERE k.chunkable_type = 'message' AND k.position = 0 AND k.account_id = ${accountPk}
+          AND k.chunkable_id IN (SELECT value FROM json_each(${JSON.stringify(pks)}))`)
+    .map(({ chatId, messageId, vector }) => ({ chatId, messageId, score: dot(query, vector) }))
+    .sort((a, b) => b.score - a.score);
 /** The account's live messages nearest to `query` by their vector of `model`, best first. */
 export const nearestMessages = (context, accountPk, { limit, query, ...filter }) => {
     const score = (row) => ({

package/dist/store/sqlite/project-transfer/records.js

@@ -1,5 +1,6 @@
 import { CliError } from "@wirecat/cli-base";
 import { fold } from "../../normalize.js";
+import { nextId } from "../memories.js";
 import { thingOf } from "../things.js";
 export const restoreAnnotationRecords = (database, projectId, snapshot, bindings) => {
     const mappings = { ...bindings };
@@ -24,8 +25,8 @@ export const restoreAnnotationRecords = (database, projectId, snapshot, bindings
         if (!author)
             throw new CliError("validation_error", "project annotation author is required");
         const row = database
-            .prepare("INSERT INTO facts(type,statement,status,project_id,subject_type,subject_id,occurred_at,supersedes_id,author_type,author_id,model,confirmed_by_type,confirmed_by_id,scope,source,created_at,updated_at) VALUES(?,?,?,?,NULL,NULL,?,NULL,?,?,?,?,?,?,?,?,?) RETURNING id")
-            .get(fact.type, fact.statement, fact.status, projectId, fact.occurred_at, author.type, author.id, fact.model, confirmed?.type ?? null, confirmed?.id ?? null, fact.scope, fact.source, fact.created_at, fact.updated_at);
+            .prepare("INSERT INTO facts(id,type,statement,status,project_id,subject_type,subject_id,occurred_at,supersedes_id,author_type,author_id,model,confirmed_by_type,confirmed_by_id,scope,source,created_at,updated_at) VALUES(?,?,?,?,?,NULL,NULL,?,NULL,?,?,?,?,?,?,?,?,?) RETURNING id")
+            .get(nextId(database, "facts"), fact.type, fact.statement, fact.status, projectId, fact.occurred_at, author.type, author.id, fact.model, confirmed?.type ?? null, confirmed?.id ?? null, fact.scope, fact.source, fact.created_at, fact.updated_at);
         if (!row)
             throw new CliError("configuration_error", "project annotation fact insertion failed");
         mappings[fact.ref] = `fact:${row.id}`;

package/dist/store/sqlite/sync.d.ts

@@ -3,7 +3,13 @@ export declare const syncStateOf: ({ orm }: StoreContext, accountKey: number, na
     value: string;
     at: string;
 } | undefined;
-export declare const writeState: ({ orm, now }: StoreContext, accountKey: number, name: string, value: string) => void;
+/** A sync state value saved with `MailStore.saveThreads`, by the account's row id. */
+export interface SyncStateSave {
+    accountId: number;
+    name: string;
+    value: string;
+}
+export declare const writeState: ({ orm, now }: Pick<StoreContext, "orm" | "now">, accountKey: number, name: string, value: string) => void;
 export declare const clearState: ({ orm }: StoreContext, accountKey: number, name: string) => void;
 /** Taken when free, expired, or already this holder's; another holder's running lease stays. */
 export declare const claim: ({ orm, now }: StoreContext, chatKey: number, anchor: string, holder: string, forMs: number) => boolean;

package/dist/store/sqlite/transaction.d.ts

@@ -11,7 +11,8 @@ export declare const write: <T>(database: CacheDatabase, body: () => T, { before
 /**
  * A write to tables that queue no derived work, in a loop that budgets its lock time or reports what it did: a fill,
  * a backfill, a rebuild. It drains nothing, so it neither does nor hides another write's work; the queues wait for
- * the next `write`.
+ * the next `write`. The outermost pauses 2 ms after it commits: a loop that begins again at once leaves another
+ * process's `write` a gap of microseconds, and it misses that gap until its 5 s wait runs out.
  */
 export declare const writeBatch: <T>(database: CacheDatabase, body: () => T) => T;
 /** Several queries that must see one state of the file. Never drains: that is the outer write's work, or nobody's. */

package/dist/store/sqlite/transaction.js

@@ -11,7 +11,8 @@ const isBusy = (error) => {
 /**
  * Takes the write lock, waiting as long as the connection's `busy_timeout`. SQLite's own wait sleeps 1 ms, then up to
  * 100 ms between tries, so a process that commits and begins again at once is never seen free and the waiter fails
- * with "database is locked" after the full timeout. Trying every millisecond sees the gap between two writes.
+ * with "database is locked" after the full timeout. Trying every millisecond sees a gap of a fraction of a
+ * millisecond between two writes; a writer that leaves none still starves it, hence `writeBatch`'s pause.
  */
 const beginImmediate = (database) => {
     const timeout = Number(database.prepare("PRAGMA busy_timeout").get()?.timeout ?? 0);
@@ -53,9 +54,17 @@ export const write = (database, body, { beforeCommit } = {}) => transact(databas
 /**
  * A write to tables that queue no derived work, in a loop that budgets its lock time or reports what it did: a fill,
  * a backfill, a rebuild. It drains nothing, so it neither does nor hides another write's work; the queues wait for
- * the next `write`.
+ * the next `write`. The outermost pauses 2 ms after it commits: a loop that begins again at once leaves another
+ * process's `write` a gap of microseconds, and it misses that gap until its 5 s wait runs out.
  */
-export const writeBatch = (database, body) => transact(database, body, false);
+export const writeBatch = (database, body) => {
+    const outer = connectionOf(database).depth === 0;
+    const result = transact(database, body, false);
+    if (outer)
+        Atomics.wait(pause, 0, 0, BATCH_PAUSE_MS);
+    return result;
+};
+const BATCH_PAUSE_MS = 2;
 const transact = (database, body, drain, beforeCommit) => {
     const connection = connectionOf(database);
     if (connection.reading)

package/dist/store/vectors.d.ts

@@ -87,6 +87,19 @@ export interface AccountVectors {
         messageId: Id;
         score: number;
     }[]>;
+    /**
+     * The account's messages among `pks` (store keys, as a query's matches give them) scored against `query` by their
+     * own vector of `model`, best first: a filter's own messages ranked, not the nearest overall.
+     */
+    scoreMessages(key: AccountKey, options: {
+        model: string;
+        query: Float32Array;
+        pks: number[];
+    }): Promise<{
+        chatId: Id;
+        messageId: Id;
+        score: number;
+    }[]>;
     /** The current build's distinct chunks, and how many have a vector of `model`. */
     vectorStatus(key: AccountKey, chatId: Id, model: string): Promise<{
         chunks: number;

package/dist/store/vectors.js

@@ -147,6 +147,10 @@ export const accountVectorsOver = (context, mail) => {
                 query,
             });
         },
+        scoreMessages: async (key, options) => {
+            const accountPk = findAccountPk(key);
+            return accountPk === undefined ? [] : messageChunks.scoreMessages(context, accountPk, options);
+        },
         vectorStatus: async (key, chatId, model) => {
             const chatKey = chatKeyOf(key, chatId);
             return chatKey === undefined ? { chunks: 0, embedded: 0 } : vectors.vectorStatus(context, chatKey, model);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.504.0",
+  "version": "0.506.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",