@wirecat/cli-core 0.553.0 → 0.555.0

package/dist/services/agenda.js

@@ -3,6 +3,7 @@ import { formatLocator, isLocator, parseLocator } from "../domain/locator.js";
 import { formatReference } from "../domain/references.js";
 import { agenda } from "../entities/agenda.js";
 import { AGENDA_COLUMNS, inboxKey, queryFilter, } from "../store/store.js";
+import { mailActivity } from "./agenda-mail.js";
 import { chatCoverage } from "./chat-coverage.js";
 import { CHAT_KINDS } from "./chats.js";
 import { taskAccount } from "./task-rules.js";
@@ -36,6 +37,10 @@ export const AGENDA_QUERY = {
     }),
     defaults: ["chat_kind:(dialog OR group)", "claimed:(none OR mine)"],
 };
+/**
+ * An email account's chats are its threads, and whose turn one is follows `mailTurn` (`../mail/owed.ts`), not merely
+ * who wrote last: mail sent to many is never owed.
+ */
 export const agendaService = (deps, now = Date.now) => ({
     show: async ({ dueWithin = DAY, waiting = AGENDA_DEFAULTS.waiting, unanswered = AGENDA_DEFAULTS.unanswered, within = 30 * DAY, query, holder, } = {}) => {
         const [store, account] = [await deps.store(), await deps.account()];
@@ -91,9 +96,12 @@ export const agendaService = (deps, now = Date.now) => ({
             createdAt: task.createdAt,
             assignees,
         }));
-        const where = parts.chats ? queryFilter(parts.chats, AGENDA_COLUMNS, { command: "tasks agenda" }) : undefined;
-        const activity = await store.chats.chatActivity(account, at - within, where);
+        const mail = account.provider === "email" ? await mailActivity(store, account, at - within, parts.chats) : undefined;
+        const activity = mail ??
+            (await store.chats.chatActivity(account, at - within, parts.chats ? queryFilter(parts.chats, AGENDA_COLUMNS, { command: "tasks agenda" }) : undefined));
+        const turnOf = (chat) => "turn" in chat ? chat.turn : chat.lastFromOwner === null ? null : chat.lastFromOwner ? "waiting" : "owed";
         const chats = activity.chats.map(({ lastMessageId, ...chat }) => ({
+            turn: turnOf({ lastMessageId, ...chat }),
             ...chat,
             lastMessage: lastMessageId === null
                 ? null
@@ -107,12 +115,14 @@ export const agendaService = (deps, now = Date.now) => ({
         }));
         const silentFor = (chat) => at - Date.parse(chat.lastMessageAt);
         const shown = {
-            owed: chats.filter((chat) => chat.lastFromOwner === false && silentFor(chat) >= unanswered),
-            waiting: chats.filter((chat) => chat.lastFromOwner === true && silentFor(chat) >= waiting),
-            unsynced: chats.filter((chat) => chat.lastMessage === null),
+            owed: chats.filter((chat) => chat.turn === "owed" && silentFor(chat) >= unanswered),
+            waiting: chats.filter((chat) => chat.turn === "waiting" && silentFor(chat) >= waiting),
+            // A thread with no stored email but drafts is not behind: mail sync stores every email it lists.
+            unsynced: mail ? [] : chats.filter((chat) => chat.lastMessage === null),
         };
-        const covered = new Map((await chatCoverage(store, account, Object.values(shown).flatMap((list) => list.map(({ chatId }) => chatId)))).map((one) => [one.chatId, one]));
-        const withCoverage = (list) => list.map((chat) => ({ ...chat, coverage: covered.get(chat.chatId) }));
+        const ids = Object.values(shown).flatMap((list) => list.map(({ chatId }) => chatId));
+        const covered = new Map((mail ? mail.coverage(ids) : await chatCoverage(store, account, ids)).map((one) => [one.chatId, one]));
+        const withCoverage = (list) => list.map(({ turn: _, ...chat }) => ({ ...chat, coverage: covered.get(chat.chatId) }));
         const inList = new Set((await store.tasks.categories()).map(({ name }) => name));
         const unlisted = new Map();
         for (const { category } of open)

package/dist/store/exports.d.ts

@@ -23,6 +23,7 @@ export { storeClients } from "./sqlite/clients.js";
 export { CHAT_LIST_KEY, type ChatCompleteness, fetchedKey, historyStartKey } from "./sqlite/completeness.js";
 export { CONVERSATION_COLUMNS } from "./sqlite/conversations.js";
 export type { CounterTarget } from "./sqlite/counters.js";
+export { type EmailMarks, MAIL_AGENDA_COLUMNS, type NewestEmail, type ThreadActivity, } from "./sqlite/email-activity.js";
 export { messageOfEmail, threadConversation } from "./sqlite/emails.js";
 export { orphanPointers } from "./sqlite/entity-types.js";
 export { involvementStoreOver } from "./sqlite/involvements.js";

package/dist/store/exports.js

@@ -6,6 +6,7 @@ export { CHAT_COLUMNS, TRACKED_CHAT_COLUMNS, titlePart } from "./sqlite/chats.js
 export { storeClients } from "./sqlite/clients.js";
 export { CHAT_LIST_KEY, fetchedKey, historyStartKey } from "./sqlite/completeness.js";
 export { CONVERSATION_COLUMNS } from "./sqlite/conversations.js";
+export { MAIL_AGENDA_COLUMNS, } from "./sqlite/email-activity.js";
 export { messageOfEmail, threadConversation } from "./sqlite/emails.js";
 export { orphanPointers } from "./sqlite/entity-types.js";
 export { involvementStoreOver } from "./sqlite/involvements.js";

package/dist/store/sqlite/email-activity.d.ts

@@ -0,0 +1,62 @@
+import type { CacheDatabase } from "../driver.js";
+import type { Where } from "./query-filter.js";
+/** What decides whose turn a thread is, read from its newest email that is not a draft; never its text. */
+export interface NewestEmail {
+    externalId: string;
+    /** ms: when it was received, or sent when that is unknown. */
+    at: number;
+    outgoing: boolean;
+    /** The mailbox's own address is in To or Cc. */
+    addressed: boolean;
+    /** Filed in the mailbox of mail sent to many. */
+    bulk: boolean;
+    /** Carries a List-Id. */
+    list: boolean;
+    /** Recorded by an envelope rule, without its text. */
+    envelope: boolean;
+    /** Its sender's mail looks like spam, a scam or a robot (the sender score). */
+    demoted: boolean;
+}
+/** One thread with mail after a time: `tasks agenda` and `review` read this, never the emails' text. */
+export interface ThreadActivity {
+    /** The thread's external id: what a `chat:email/…` reference names. */
+    threadId: string;
+    subject: string | null;
+    /** ms. */
+    lastEmailAt: number;
+    emails: number;
+    /** `null` when every email of the thread is a draft. */
+    newest: NewestEmail | null;
+}
+/** What a reader sorts mail by, per email: never its text. */
+export interface EmailMarks {
+    /** Filed in the mailbox of mail sent to many. */
+    bulk: boolean;
+    listId: string | null;
+    /** The mail rule that decided how it is stored, by its id in `rules list`. */
+    ruleId: number | null;
+}
+export interface MailActivityStore {
+    /** Each live email's marks, by its Message-ID; one the store does not hold is left out. */
+    emailMarks(accountId: number, externalIds: readonly string[]): Promise<Map<string, EmailMarks>>;
+    /**
+     * Each live thread of the account whose newest email is after `since` (ms), newest first, with the facts of that
+     * email. `where` is a `tasks agenda` query compiled over `MAIL_AGENDA_COLUMNS`.
+     */
+    threadActivity(accountId: number, since: number, where?: Where): Promise<ThreadActivity[]>;
+}
+/** `tasks agenda` fields over `threadActivity`'s threads: an email thread counts as a dialog. */
+export declare const MAIL_AGENDA_COLUMNS: {
+    readonly chat_kind: {
+        readonly type: "enum";
+        readonly sql: "'dialog'";
+    };
+};
+export declare const threadActivity: (database: CacheDatabase, accountId: number, since: number, where?: Where) => ThreadActivity[];
+/** A thread's live emails, oldest first; `window` keeps those received in (after, until], in ms. */
+export declare const threadEmailIds: (database: CacheDatabase, accountId: number, thread: string, { after, until }?: {
+    after?: number;
+    until?: number;
+}) => number[];
+export declare const emailMarks: (database: CacheDatabase, accountId: number, externalIds: readonly string[]) => Map<string, EmailMarks>;
+export declare const mailActivity: (database: CacheDatabase) => MailActivityStore;

package/dist/store/sqlite/email-activity.js

@@ -0,0 +1,71 @@
+import { int, str } from "./events.js";
+/** `tasks agenda` fields over `threadActivity`'s threads: an email thread counts as a dialog. */
+export const MAIL_AGENDA_COLUMNS = { chat_kind: { type: "enum", sql: "'dialog'" } };
+// The thread's own time is the Date header where sync stored one, so a thread is picked by it; the newest email is
+// picked by when it arrived, which is what an answer follows.
+const ACTIVITY = `
+  SELECT t.external_id AS thread, t.subject, t.last_email_at, t.emails_count,
+    e.external_id, coalesce(e.received_at, e.sent_at) AS at,
+    coalesce(e.outgoing, lower(e.from_address) = lower(a.external_id)) AS outgoing,
+    EXISTS (SELECT 1 FROM email_recipients r WHERE r.email_id = e.id AND r.role IN ('to', 'cc')
+      AND lower(r.address) = lower(a.external_id)) AS addressed,
+    EXISTS (SELECT 1 FROM email_mailboxes m JOIN mailboxes b ON b.id = m.mailbox_id
+      WHERE m.email_id = e.id AND b.kind = 'bulk') AS bulk,
+    coalesce(json_extract(e.headers, '$."list-id"'), json_extract(e.metadata, '$.listId')) IS NOT NULL AS list,
+    EXISTS (SELECT 1 FROM email_mailboxes m JOIN mailboxes b ON b.id = m.mailbox_id
+      WHERE m.email_id = e.id AND b.kind = 'envelope') AS envelope,
+    EXISTS (SELECT 1 FROM account_identities ai WHERE ai.account_id = t.account_id
+      AND ai.identity_id = e.from_identity_id AND ai.mail_demoted = 1) AS demoted
+  FROM email_threads t JOIN accounts a ON a.id = t.account_id
+  LEFT JOIN emails e ON e.id = (SELECT f.id FROM emails f WHERE f.email_thread_id = t.id AND f.deleted_at IS NULL
+    AND coalesce(f.draft, 0) = 0 ORDER BY coalesce(f.received_at, f.sent_at) DESC, f.id DESC LIMIT 1)
+  WHERE t.account_id = ? AND t.deleted_at IS NULL AND t.last_email_at > ?`;
+export const threadActivity = (database, accountId, since, where) => database
+    .prepare(`${ACTIVITY}${where ? ` AND (${where.sql})` : ""} ORDER BY t.last_email_at DESC, t.id DESC`)
+    .all(accountId, since, ...(where?.params ?? []))
+    .map((row) => ({
+    threadId: String(row.thread),
+    subject: str(row.subject),
+    lastEmailAt: Number(row.last_email_at),
+    emails: Number(row.emails_count),
+    newest: row.external_id == null
+        ? null
+        : {
+            externalId: String(row.external_id),
+            at: int(row.at) ?? Number(row.last_email_at),
+            outgoing: Boolean(row.outgoing),
+            addressed: Boolean(row.addressed),
+            bulk: Boolean(row.bulk),
+            list: Boolean(row.list),
+            envelope: Boolean(row.envelope),
+            demoted: Boolean(row.demoted),
+        },
+}));
+/** A thread's live emails, oldest first; `window` keeps those received in (after, until], in ms. */
+export const threadEmailIds = (database, accountId, thread, { after, until } = {}) => database
+    .prepare(`SELECT e.id FROM emails e JOIN email_threads t ON t.id = e.email_thread_id
+        WHERE t.account_id = ? AND t.external_id = ? AND e.deleted_at IS NULL
+          AND coalesce(e.received_at, e.sent_at, 0) > coalesce(?, -1)
+          AND coalesce(e.received_at, e.sent_at, 0) <= coalesce(?, 9e15)
+        ORDER BY coalesce(e.sent_at, e.received_at), e.id`)
+    .all(accountId, thread, after ?? null, until ?? null)
+    .map(({ id }) => Number(id));
+const MARKS = `
+  SELECT e.mail_rule_id, coalesce(json_extract(e.headers, '$."list-id"'), json_extract(e.metadata, '$.listId')) AS list_id,
+    EXISTS (SELECT 1 FROM email_mailboxes m JOIN mailboxes b ON b.id = m.mailbox_id
+      WHERE m.email_id = e.id AND b.kind = 'bulk') AS bulk
+  FROM emails e WHERE e.account_id = ? AND e.external_id = ? AND e.deleted_at IS NULL`;
+export const emailMarks = (database, accountId, externalIds) => {
+    const one = database.prepare(MARKS);
+    const marks = new Map();
+    for (const id of externalIds) {
+        const row = one.get(accountId, id);
+        if (row)
+            marks.set(id, { bulk: Boolean(row.bulk), listId: str(row.list_id), ruleId: int(row.mail_rule_id) });
+    }
+    return marks;
+};
+export const mailActivity = (database) => ({
+    emailMarks: async (accountId, externalIds) => emailMarks(database, accountId, externalIds),
+    threadActivity: async (accountId, since, where) => threadActivity(database, accountId, since, where),
+});

package/dist/store/sqlite/email-locations.d.ts

@@ -59,3 +59,4 @@ export declare const partialMail: (database: CacheDatabase) => PartialMailStore;
 export declare const storedWhole: (database: CacheDatabase, emailId: number) => boolean;
 export declare const setLocations: (database: CacheDatabase, accountId: number, folder: string, uidvalidity: string, found: ReadonlyMap<string, number[]>) => number;
 export declare const forgetLocations: (database: CacheDatabase, accountId: number, folder: string, uidvalidity: string, uids: number[]) => void;
+export declare const backfillSentAt: (database: CacheDatabase, accountId: number, limit: number) => number;

package/dist/store/sqlite/email-locations.js

@@ -107,3 +107,7 @@ export const forgetLocations = (database, accountId, folder, uidvalidity, uids)
     for (const emailId of new Set(emails))
         leave.run(emailId, Number(box.id), emailId, Number(box.id));
 };
+export const backfillSentAt = (database, accountId, limit) => Number(database
+    .prepare(`UPDATE emails SET sent_at = received_at WHERE id IN (
+           SELECT id FROM emails WHERE account_id = ? AND sent_at IS NULL AND received_at IS NOT NULL LIMIT ?)`)
+    .run(accountId, limit).changes);

package/dist/store/sqlite/emails.d.ts

@@ -3,6 +3,7 @@ import type { Attachment, Metadata } from "../../meetings/index.js";
 import type { Stemmer, Stemmers } from "../../search/stem.js";
 import type { CacheDatabase } from "../driver.js";
 import type { ConversationSummary, StoredHit } from "../types.js";
+import { type MailActivityStore } from "./email-activity.js";
 import { type EmailLocationInput, type FolderEmail, type PartialMailStore } from "./email-locations.js";
 import { type MailRulesStore } from "./mail-rules.js";
 import { type AttachmentInput } from "./meetings.js";
@@ -205,7 +206,7 @@ export interface SenderScore {
     demoted?: boolean;
 }
 /** Mail in tables of its own: threads, emails, recipients and mailboxes, keyed by account and Message-ID. */
-export interface MailStore extends PartialMailStore {
+export interface MailStore extends PartialMailStore, MailActivityStore {
     saveThread(input: ThreadSave): Promise<ThreadDetails>;
     /** Several threads and sync state in one transaction: a slice of mail commits with its resume point, or not at all. */
     saveThreads(inputs: ThreadSave[], states?: SyncStateSave[]): Promise<ThreadDetails[]>;
@@ -238,6 +239,8 @@ export interface MailStore extends PartialMailStore {
         read?: boolean;
         flagged?: boolean;
     }>, now: number): Promise<number>;
+    /** Gives up to `limit` emails stored without `sent_at` their internal date; answers how many. */
+    backfillSentAt(accountId: number, limit: number): Promise<number>;
     /** The live emails whose location in `folder` has this UIDVALIDITY, oldest UID first. */
     folderEmails(accountId: number, folder: string, uidvalidity: string): Promise<FolderEmail[]>;
     /**
@@ -266,8 +269,11 @@ export interface MailStore extends PartialMailStore {
     }[]>;
     /** Live emails of one account by Message-ID, each placed in its thread; an unknown one is left out. */
     inThreads(accountId: number, externalIds: string[]): Promise<EmailInThread[]>;
-    /** A thread's live emails, oldest first, as search hits: whole text, people, labels and locator. */
-    threadHits(accountId: number, threadExternalId: string): Promise<StoredHit[]>;
+    /** A thread's live emails, oldest first, as search hits; `window` keeps those received in (after, until], in ms. */
+    threadHits(accountId: number, thread: string, window?: {
+        after?: number;
+        until?: number;
+    }): Promise<StoredHit[]>;
     /** The emails of one account nearest in meaning to `query`, each by its best chunk, best first. */
     nearest(model: string, query: Float32Array, options: {
         accountId: number;

package/dist/store/sqlite/emails.js

@@ -1,8 +1,9 @@
 import { formatLocator } from "../../domain/locator.js";
 import { formatReference } from "../../domain/references.js";
 import { fold, normalize } from "../normalize.js";
+import { mailActivity, threadEmailIds } from "./email-activity.js";
 import { chunkAttachments, drainEmailIndex, dropAttachmentChunks, MAIL_PIECES, queueThreads, threadPieces, titleOf, } from "./email-index.js";
-import { folderEmails, forgetLocations, locate, partialMail, setLocations, storedWhole, } from "./email-locations.js";
+import { backfillSentAt, folderEmails, forgetLocations, locate, partialMail, setLocations, storedWhole, } from "./email-locations.js";
 import { markEmailsDeleted, markEmailsRestored, recountEmailThread as recount, setEmailFlags } from "./email-state.js";
 import { bool, flag, fromJson, int, page, str, toJson } from "./events.js";
 import { folded } from "./folded.js";
@@ -472,6 +473,7 @@ export const mailStoreOver = ({ database, orm, now }, stemmerFor = stemmerCache(
         }));
     },
     ...partialMail(database),
+    ...mailActivity(database),
     async emailFacts(filter = {}) {
         const [where, params] = emailWhere(filter);
         return database
@@ -554,11 +556,8 @@ export const mailStoreOver = ({ database, orm, now }, stemmerFor = stemmerCache(
             return count;
         });
     },
-    async setFlags(accountId, flags, now) {
-        return write(database, () => {
-            return setEmailFlags(database, accountId, flags, now);
-        });
-    },
+    setFlags: async (accountId, flags, now) => write(database, () => setEmailFlags(database, accountId, flags, now)),
+    backfillSentAt: async (accountId, limit) => write(database, () => backfillSentAt(database, accountId, limit)),
     folderEmails: async (accountId, folder, uidvalidity) => folderEmails(database, accountId, folder, uidvalidity),
     setLocations: async (accountId, folder, uidvalidity, found) => write(database, () => setLocations(database, accountId, folder, uidvalidity, found)),
     forgetLocations: async (accountId, folder, uidvalidity, uids) => write(database, () => forgetLocations(database, accountId, folder, uidvalidity, uids)),
@@ -634,14 +633,8 @@ export const mailStoreOver = ({ database, orm, now }, stemmerFor = stemmerCache(
             return place ? [{ ...place, score }] : [];
         });
     },
-    async threadHits(accountId, threadExternalId) {
-        const ids = database
-            .prepare(`SELECT e.id FROM emails e JOIN email_threads t ON t.id = e.email_thread_id
-          WHERE t.account_id = ? AND t.external_id = ? AND e.deleted_at IS NULL
-          ORDER BY coalesce(e.sent_at, e.received_at), e.id`)
-            .all(accountId, threadExternalId)
-            .map(({ id }) => Number(id));
-        return emailHitsByPk(database, ids);
+    async threadHits(accountId, thread, window) {
+        return emailHitsByPk(database, threadEmailIds(database, accountId, thread, window));
     },
     async inThreads(accountId, externalIds) {
         const id = database.prepare("SELECT id FROM emails WHERE account_id = ? AND external_id = ? AND deleted_at IS NULL");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.553.0",
+  "version": "0.555.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",