@wirecat/cli-core 0.491.0 → 0.493.0

package/dist/services/agenda.js

@@ -3,6 +3,7 @@ import { formatLocator, isLocator, parseLocator } from "../domain/locator.js";
 import { formatReference } from "../domain/references.js";
 import { agenda } from "../entities/agenda.js";
 import { AGENDA_COLUMNS, inboxKey, queryFilter, } from "../store/store.js";
+import { chatCoverage } from "./chat-coverage.js";
 import { CHAT_KINDS } from "./chats.js";
 import { taskAccount } from "./task-rules.js";
 import { taskTagFilters } from "./task-tag-filters.js";
@@ -102,21 +103,29 @@ export const agendaService = (deps, now = Date.now) => ({
                 }),
             openTasks: open.filter((task) => chatOf(task.source) === chat.chatId).map((task) => task.id),
         }));
+        const silentFor = (chat) => at - Date.parse(chat.lastMessageAt);
+        const shown = {
+            owed: chats.filter((chat) => chat.lastFromOwner === false && silentFor(chat) >= unanswered),
+            waiting: chats.filter((chat) => chat.lastFromOwner === true && silentFor(chat) >= waiting),
+            unsynced: chats.filter((chat) => chat.lastMessage === null),
+        };
+        const covered = new Map((await chatCoverage(store, account, Object.values(shown).flatMap((list) => list.map(({ chatId }) => chatId)))).map((one) => [one.chatId, one]));
+        const withCoverage = (list) => list.map((chat) => ({ ...chat, coverage: covered.get(chat.chatId) }));
         const inList = new Set((await store.tasks.categories()).map(({ name }) => name));
         const unlisted = new Map();
         for (const { category } of open)
             if (category && !inList.has(category))
                 unlisted.set(category, (unlisted.get(category) ?? 0) + 1);
-        const silentFor = (chat) => at - Date.parse(chat.lastMessageAt);
         return {
             at: new Date(at).toISOString(),
+            since: new Date(at - within).toISOString(),
             account: taskAccount(account),
             storedAt: activity.storedAt,
             stale: activity.storedAt === null || at - Date.parse(activity.storedAt) > DAY,
             tasks,
-            owed: chats.filter((chat) => chat.lastFromOwner === false && silentFor(chat) >= unanswered),
-            waiting: chats.filter((chat) => chat.lastFromOwner === true && silentFor(chat) >= waiting),
-            unsynced: chats.filter((chat) => chat.lastMessage === null),
+            owed: withCoverage(shown.owed),
+            waiting: withCoverage(shown.waiting),
+            unsynced: withCoverage(shown.unsynced),
             assigned,
             unlistedCategories: [...unlisted].sort().map(([name, openTasks]) => ({ name, openTasks })),
         };
@@ -208,6 +217,18 @@ export const agendaItem = (shown) => {
         last_message_at: one.lastMessageAt,
         last_from_owner: one.lastFromOwner,
         open_tasks: one.openTasks,
+        completeness: {
+            state: one.coverage.completeness.state,
+            up_to_date: one.coverage.completeness.upToDate,
+            gaps: one.coverage.completeness.gaps,
+            reaches_start: one.coverage.completeness.reachesStart,
+            fetched_at: one.coverage.completeness.fetchedAt,
+        },
+        held: {
+            messages: one.coverage.held.messages,
+            oldest_at: one.coverage.held.oldestAt,
+            newest_at: one.coverage.held.newestAt,
+        },
     });
     return {
         at: shown.at,

package/dist/services/chat-coverage.d.ts

@@ -0,0 +1,19 @@
+import type { Id } from "../domain/models.js";
+import type { AccountKey, ChatCompleteness, MessageStore } from "../store/store.js";
+/** What the store holds of one chat, and how far that goes: never message text. */
+export interface ChatCoverage {
+    chatId: Id;
+    /** `null` while the store holds no message of the chat: titles come with the stats. */
+    title: string | null;
+    completeness: ChatCompleteness;
+    held: {
+        messages: number;
+        oldestAt: string | null;
+        newestAt: string | null;
+    };
+}
+/**
+ * Per chat, in the order asked: `chatCompleteness` with the span `chatStats` holds. Stats are read one chat at a time,
+ * because the account-wide read groups every message of every chat. A chat the store has no row for is `unknown`.
+ */
+export declare const chatCoverage: (store: Pick<MessageStore, "chats">, account: AccountKey, chatIds: Id[]) => Promise<ChatCoverage[]>;

package/dist/services/chat-coverage.js

@@ -0,0 +1,27 @@
+/**
+ * Per chat, in the order asked: `chatCompleteness` with the span `chatStats` holds. Stats are read one chat at a time,
+ * because the account-wide read groups every message of every chat. A chat the store has no row for is `unknown`.
+ */
+export const chatCoverage = async (store, account, chatIds) => {
+    const ids = [...new Set(chatIds)];
+    if (ids.length === 0)
+        return [];
+    const found = new Map((await store.chats.chatCompleteness(account, ids)).map((one) => [one.chatId, one]));
+    return Promise.all(ids.map(async (chatId) => {
+        const [stats] = await store.chats.chatStats(account, chatId);
+        const completeness = found.get(chatId) ?? {
+            chatId,
+            state: "unknown",
+            upToDate: null,
+            gaps: false,
+            reachesStart: false,
+            fetchedAt: null,
+        };
+        return {
+            chatId,
+            title: stats?.title ?? null,
+            completeness,
+            held: { messages: stats?.messages ?? 0, oldestAt: stats?.oldestAt ?? null, newestAt: stats?.newestAt ?? null },
+        };
+    }));
+};

package/dist/services/messages-discovery.d.ts

@@ -12,6 +12,12 @@ export declare const RERANK_DEPTH = 50;
  * ~7–10 s of it (cli-testing performance/search/evidence-pipeline, "In Core").
  */
 export declare const RERANK_MILLISECONDS = 15000;
+/**
+ * Follow-ups: each of the first 5 hits gets up to 2 later messages that share a word at most 3 messages hold. A
+ * blind benchmark split found 4 more of 15 cancelled decisions this way and lost no answer (cli-testing
+ * performance/search/evidence-pipeline, "Cancelled decisions and long messages").
+ */
+export declare const FOLLOW_UP_HITS = 5;
 /** The reranker, opened; or why discovery keeps its own order. */
 export type RerankLane = () => Promise<{
     model: string;

package/dist/services/messages-discovery.js

@@ -18,11 +18,23 @@ const RERANK_CHUNK = 5;
  * ~7–10 s of it (cli-testing performance/search/evidence-pipeline, "In Core").
  */
 export const RERANK_MILLISECONDS = 15_000;
+/**
+ * Follow-ups: each of the first 5 hits gets up to 2 later messages that share a word at most 3 messages hold. A
+ * blind benchmark split found 4 more of 15 cancelled decisions this way and lost no answer (cli-testing
+ * performance/search/evidence-pipeline, "Cancelled decisions and long messages").
+ */
+export const FOLLOW_UP_HITS = 5;
+const FOLLOW_UPS_PER_HIT = 2;
+const RARE_AT = 3;
+/** Later messages read per hit before the search's filters; the latest that pass are kept. */
+const FOLLOW_UP_CANDIDATES = 20;
 export const searchDiscovery = async (store, account, asked, messenger = {}, meaning, rerank) => {
-    // A question's own "?" would parse as a one-character wildcard and send it to strict search (BUG-395).
+    // A question's own "?" would parse as a one-character wildcard and send it to strict search (BUG-395); the
+    // colon of a time or ratio ("22:00", "1:1") would parse as a field and fail the whole search, and escaped it is
+    // syntax again, so it splits the numbers (BUG-450).
     const request = asked.ast !== undefined || asked.exact || !asked.text
         ? asked
-        : { ...asked, text: asked.text.replace(/(?<=\p{L})\?(?=\s|$)/gu, "") };
+        : { ...asked, text: asked.text.replace(/(?<=\p{L})\?(?=\s|$)/gu, "").replace(/(?<=\d):(?=\d)/gu, " ") };
     const original = await prepareLucene(store, account, request, messenger);
     const text = request.text ?? "";
     const plan = planQuestion(text, request.ast !== undefined || !!request.exact || !!request.newest);
@@ -208,7 +220,10 @@ export const searchDiscovery = async (store, account, asked, messenger = {}, mea
         started += performance.now() - opening;
     }
     check();
-    const items = await Promise.all(candidates.slice(0, request.limit).map(async ({ fused: _fused, ...m }) => {
+    const followUps = await followUpsOf(store, account, request, messenger, candidates, plan.scope);
+    const items = await Promise.all(candidates.slice(0, request.limit).map(async ({ fused: _fused, ...hit }) => {
+        const later = followUps.get(hit.locator);
+        const m = later?.length ? { ...hit, followUps: later } : hit;
         if (!request.context)
             return m;
         check();
@@ -235,6 +250,7 @@ export const searchDiscovery = async (store, account, asked, messenger = {}, mea
                 candidateDepth: 300,
                 candidates: candidates.length,
                 repliesAdded: replyCount,
+                followUps: [...followUps.values()].reduce((sum, later) => sum + later.length, 0),
                 truncated,
                 terms,
                 queries: [...(plan.changed ? [] : plan.queries), relaxed],
@@ -247,6 +263,37 @@ export const searchDiscovery = async (store, account, asked, messenger = {}, mea
         ...(lane && "partly" in lane && lane.partly ? { meaningSkipped: lane.partly } : {}),
     };
 };
+/**
+ * The follow-ups of the first hits that the question's filters allow, by hit. A message already shown above — a hit
+ * or an earlier hit's follow-up — is not shown again, as the benchmark skipped it.
+ */
+const followUpsOf = async (store, account, request, messenger, candidates, scope) => {
+    const head = candidates.slice(0, Math.min(FOLLOW_UP_HITS, request.limit));
+    const later = await store.messages.laterSharingRare(head.map((hit) => ({ account: accountOfHit(hit), chatId: hit.chatId, id: hit.id })), { rareAt: RARE_AT, limit: FOLLOW_UP_CANDIDATES });
+    const only = [...new Map(later.flat().map((m) => [m.locator, m])).values()].map((m) => ({
+        ...accountOfHit(m),
+        chatId: m.chatId,
+        id: m.id,
+    }));
+    const out = new Map();
+    if (!only.length)
+        return out;
+    const eligible = await searchLucene(store, account, { ...request, text: scope || undefined, ast: undefined, limit: only.length, only, context: 0, thread: undefined }, messenger);
+    const allowed = new Map(eligible.items.map((m) => [m.locator, m]));
+    const shown = new Set();
+    head.forEach((hit, i) => {
+        shown.add(hit.locator);
+        const picked = (later[i] ?? [])
+            .flatMap((m) => allowed.get(m.locator) ?? [])
+            .slice(0, FOLLOW_UPS_PER_HIT)
+            .filter((m) => !shown.has(m.locator));
+        for (const m of picked)
+            shown.add(m.locator);
+        if (picked.length)
+            out.set(hit.locator, picked);
+    });
+    return out;
+};
 /** The nearest messages the question's filters allow, as hits, nearest first. */
 export const nearEligible = async (store, account, request, messenger, ranked, scope) => {
     const keyOf = (one) => JSON.stringify([one.provider, one.account, one.chatId, one.id]);

package/dist/services/messages-search.d.ts

@@ -12,6 +12,8 @@ export interface QueryMetadata {
         candidateDepth: number;
         candidates: number;
         repliesAdded: number;
+        /** Later messages shown with the first hits because they share a rare word with one. */
+        followUps: number;
         truncated: boolean;
         terms: string[];
         queries: string[];

package/dist/services/messages.d.ts

@@ -97,6 +97,8 @@ export type FoundMessage = StoredHit & {
     exact?: boolean;
     source?: HitSource;
     context?: WindowedMessage[];
+    /** A discovery hit's later messages that share a rare word with it, latest first; one may change or cancel it. */
+    followUps?: FoundMessage[];
     thread?: ThreadContext;
 };
 export interface SearchFound extends Page<FoundMessage> {

package/dist/services/search-all.d.ts

@@ -4,6 +4,7 @@ import type { MessengerTraits } from "../ports/traits.js";
 import type { AccountKey, MessageStore } from "../store/store.js";
 import type { SearchFound, SearchQuery } from "./messages.js";
 import { type MeaningLane, type MeaningSkipped } from "./messages-discovery.js";
+import type { SearchCoverage } from "./messages-search.js";
 import type { Backend, ServerOptions, ServerSearched } from "./server-search.js";
 export declare const RESOURCES_SEARCHED: readonly ["messages", "mail", "notes", "events"];
 export type SearchedResource = (typeof RESOURCES_SEARCHED)[number];
@@ -77,6 +78,8 @@ export interface SearchAllFound {
     };
     /** The messenger's server step for messages, as `search messages` reports it. */
     server?: ServerSearched;
+    /** What the store held of the chats the messages were searched in; absent when messages were not searched. */
+    coverage?: SearchCoverage;
 }
 export interface SearchAllRequest {
     text: string;

package/dist/services/search-all.js

@@ -53,6 +53,7 @@ export const searchAll = async (store, account, request, messenger = {}, searchM
     let events;
     let messagesBy;
     let server;
+    let coverage;
     const messagesOf = async (kind) => {
         const given = request.scope?.[kind];
         const query = {
@@ -76,6 +77,8 @@ export const searchAll = async (store, account, request, messenger = {}, searchM
         const found = kind === "messages" ? await searchMessages(query) : await searchStore(store, account, query, messenger);
         if (found.server)
             server = found.server;
+        if (kind === "messages" && found.coverage)
+            coverage = found.coverage;
         hasMore ||= found.hasMore;
         const read = found.coverage?.accounts ??
             given ??
@@ -243,5 +246,6 @@ export const searchAll = async (store, account, request, messenger = {}, searchM
         ...(events === undefined ? {} : { events }),
         ...(messagesBy === undefined ? {} : { messages: messagesBy }),
         ...(server === undefined ? {} : { server }),
+        ...(coverage === undefined ? {} : { coverage }),
     };
 };

package/dist/store/messages.d.ts

@@ -157,6 +157,15 @@ export interface MessagesStore extends AccountMessages {
         chatId: Id;
         id: Id;
     }[], limit: number): Promise<Page<StoredHit>>;
+    /** Per message, later messages sharing a word at most `rareAt` messages hold, latest first; empty until stems are built. */
+    laterSharingRare(wanted: {
+        account: AccountKey;
+        chatId: Id;
+        id: Id;
+    }[], options: {
+        rareAt: number;
+        limit: number;
+    }): Promise<StoredHit[][]>;
 }
 export declare const accountMessagesOver: (context: StoreContext) => AccountMessages;
 export declare const messagesStoreOver: (context: StoreContext, keyed: AccountMessages) => MessagesStore;

package/dist/store/messages.js

@@ -1,6 +1,7 @@
 import { keysOver } from "./keys.js";
 import { applyCounterObservations } from "./sqlite/counters.js";
 import { documentLinkMessages } from "./sqlite/document-links.js";
+import { laterSharingRare } from "./sqlite/follow-ups.js";
 import * as messageWrites from "./sqlite/messages.js";
 import * as personLinks from "./sqlite/person-links.js";
 import * as reads from "./sqlite/reads.js";
@@ -144,4 +145,5 @@ export const messagesStoreOver = (context, keyed) => ({
     search: async (query, { limit, account }) => search.find(context, { text: query, limit, ...(account ? { account } : {}) }),
     find: async (filter) => search.find(context, filter),
     directReplies: async (parents, limit) => search.directReplies(context, parents, limit),
+    laterSharingRare: async (wanted, options) => laterSharingRare(context, wanted, options),
 });

package/dist/store/sqlite/follow-ups.d.ts

@@ -0,0 +1,17 @@
+import type { AccountKey, StoredHit } from "../types.js";
+import type { StoreContext } from "./open.js";
+/**
+ * For each message, the later messages of the store that share a rare word with it, latest first, at most `limit`
+ * each. A word is rare when at most `rareAt` messages hold its stem. A message that cancels or changes a decision
+ * often names the same supplier, code or term without replying to it (cli-testing
+ * performance/search/evidence-pipeline, "Cancelled decisions and long messages"). Empty while the stems are not
+ * built, since their counts would be short.
+ */
+export declare const laterSharingRare: (context: StoreContext, wanted: {
+    account: AccountKey;
+    chatId: string;
+    id: string;
+}[], { rareAt, limit }: {
+    rareAt: number;
+    limit: number;
+}) => StoredHit[][];

package/dist/store/sqlite/follow-ups.js

@@ -0,0 +1,59 @@
+import { CliError } from "@wirecat/cli-base";
+import { hitsByPk } from "./search.js";
+import { stemmerCache, stemsState, storeStemmers } from "./stems.js";
+const stemmerFor = stemmerCache();
+/**
+ * For each message, the later messages of the store that share a rare word with it, latest first, at most `limit`
+ * each. A word is rare when at most `rareAt` messages hold its stem. A message that cancels or changes a decision
+ * often names the same supplier, code or term without replying to it (cli-testing
+ * performance/search/evidence-pipeline, "Cancelled decisions and long messages"). Empty while the stems are not
+ * built, since their counts would be short.
+ */
+export const laterSharingRare = (context, wanted, { rareAt, limit }) => {
+    if (wanted.length > 50 || !Number.isInteger(limit) || limit < 1 || limit > 100)
+        throw new CliError("validation_error", "a follow-up lookup takes at most 50 messages and 1–100 results each");
+    const { database } = context;
+    const stemmers = storeStemmers(database);
+    if (!wanted.length || !stemmers || !stemsState(database)?.ready)
+        return wanted.map(() => []);
+    const stemmer = stemmerFor(stemmers);
+    // A temporary table: reading the index's own counts needs no migration, and it leaves with the connection.
+    database.exec("CREATE VIRTUAL TABLE IF NOT EXISTS temp.message_stem_counts USING fts5vocab(main, message_stems, col)");
+    const count = database.prepare("SELECT doc FROM temp.message_stem_counts WHERE term = ? AND col = 'stems'");
+    const counts = new Map();
+    const countOf = (stem) => {
+        let found = counts.get(stem);
+        if (found === undefined) {
+            found = Number(count.get(stem)?.doc ?? 0);
+            counts.set(stem, found);
+        }
+        return found;
+    };
+    const message = database.prepare(`
+    SELECT m.text, m.sent_at FROM accounts ac
+    JOIN chats c ON c.account_id = ac.id AND c.external_id = ?
+    JOIN messages m ON m.chat_id = c.id AND m.external_id = ?
+    WHERE ac.provider = ? AND ac.external_id = ? AND m.deleted_at IS NULL
+  `);
+    const later = database.prepare(`
+    SELECT m.id FROM message_stems s JOIN messages m ON m.id = s.rowid
+    WHERE message_stems MATCH ? AND m.deleted_at IS NULL AND m.sent_at > ?
+    ORDER BY m.sent_at DESC, m.id DESC LIMIT ?
+  `);
+    return wanted.map(({ account, chatId, id }) => {
+        const row = message.get(chatId, id, account.provider, account.account);
+        if (!row)
+            return [];
+        // Words of three letters or more, as the benchmark took them: a number or a code with digits is not a name here.
+        const stems = new Set(stemmer
+            .explain(String(row.text))
+            .filter(({ word }) => /^\p{L}{3,}$/u.test(word))
+            .map(({ word }) => stemmer.stemToken(word))
+            .filter(Boolean));
+        const rare = [...stems].filter((stem) => countOf(stem) <= rareAt);
+        if (!rare.length)
+            return [];
+        const match = `stems : (${rare.map((stem) => `"${stem.replaceAll('"', '""')}"`).join(" OR ")})`;
+        return hitsByPk(context, later.all(match, Number(row.sent_at), limit).map((one) => Number(one.id)));
+    });
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.491.0",
+  "version": "0.493.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",