@withbuddi/buddi 0.1.0-pre.27 → 0.1.0-pre.29

package/packages/tools/email/dist/sources/inbox-poll.js

@@ -1,11 +1,17 @@
-import { INBOX, listAccounts, markAccountSynced } from '../config.js';
+import { findAccount, INBOX, listAccounts, markAccountSynced } from '../config.js';
 import { mailboxAuth } from '../credentials.js';
-import { planFolders } from '../folders.js';
+import { clearLoginFailure, isAuthFailure, recordLoginFailure } from '../logins.js';
+import { applyPendingArrivals, oweArrivalAction } from '../mailbox/arrival.js';
+import { reconcilePendingChanges } from '../mailbox/actions.js';
+import { placeWords, planFolders } from '../folders.js';
 import { prepareForIngest, triagePrompt } from '../mail.js';
 import { scanMessageDates, skipDates } from '../dates-store.js';
 import { applyPolicies } from '../policies/gate.js';
 import { loadPolicies, recordEvent, settleEvent } from '../policies/store.js';
 import { ownerReplies, senderVerdicts } from '../policies/learn.js';
+import { sweepOpenBulkCards, tellOwnerLearned } from '../policies/auto.js';
+import { IDLE_SLOW_POLL_SECONDS, IdleWatchers } from './idle.js';
+import { checkPresence } from './presence.js';
 import { FOLDER_COLUMNS, toFolder } from '../rows.js';
 import { findThread, joinThread, threadMessages } from '../threads.js';
 import { PROCESSING_VERSION } from '../tools/shared.js';
@@ -27,6 +33,11 @@ export const MAX_PER_POLL = 50;
 export const FLAG_SYNC_WINDOW = 2_000;
 /** Uids per `UID FETCH … FLAGS`, so one command line stays a sane length. */
 export const FLAG_SYNC_BATCH = 500;
+/**
+ * Older mail whose `bulk` was never read from its headers, re-read per poll:
+ * at most this many messages, newest first, in one header-only fetch.
+ */
+export const BULK_BACKFILL_PER_POLL = 200;
 /** Deadline for every single IMAP call. Env: `EMAIL_POLL_TIMEOUT_MS`. */
 export const DEFAULT_POLL_TIMEOUT_MS = 45_000;
 export const POLL_TIMEOUT_VAR = 'EMAIL_POLL_TIMEOUT_MS';
@@ -100,12 +111,13 @@ export function triageDedupKey(messageRowId) {
  * LIST/STATUS fills an uninitialised cursor but never replaces one already in
  * use.
  */
-async function ensureFolder(db, account, name, kind = 'other', synced = false, boundary) {
-    const { rows } = await db.query(`insert into email.folders (account_id, name, kind, synced, uidvalidity, last_uid)
-     values ($1, $2, $3, $4, $5, $6)
+async function ensureFolder(db, account, name, kind = 'other', synced = false, boundary, specialUse = null) {
+    const { rows } = await db.query(`insert into email.folders (account_id, name, kind, synced, uidvalidity, last_uid, special_use)
+     values ($1, $2, $3, $4, $5, $6, $7)
      on conflict (account_id, name) do update
         set kind = excluded.kind,
             synced = excluded.synced,
+            special_use = coalesce(excluded.special_use, email.folders.special_use),
             uidvalidity = case
               when email.folders.uidvalidity is null and excluded.kind = 'sent'
                 then excluded.uidvalidity
@@ -116,7 +128,7 @@ async function ensureFolder(db, account, name, kind = 'other', synced = false, b
                 then excluded.last_uid
               else email.folders.last_uid
             end
-     returning ${FOLDER_COLUMNS}`, [account.id, name, kind, synced, boundary?.uidValidity ?? null, boundary?.lastUid ?? 0]);
+     returning ${FOLDER_COLUMNS}`, [account.id, name, kind, synced, boundary?.uidValidity ?? null, boundary?.lastUid ?? 0, specialUse]);
     const row = rows[0];
     if (!row)
         throw new Error('email.inbox-poll: folder upsert returned no row');
@@ -174,7 +186,11 @@ async function discoverFolders(db, account, client, timeoutMs, log) {
             }
             : undefined;
         try {
-            await ensureFolder(db, account, folder.name, folder.kind, folder.synced, boundary);
+            const use = listed
+                ? [listed.specialUse ?? '', ...listed.flags].map((f) => f.trim().toLowerCase())
+                    .find((f) => ['\\all', '\\archive', '\\trash', '\\junk', '\\sent', '\\drafts'].includes(f)) ?? null
+                : null;
+            await ensureFolder(db, account, folder.name, folder.kind, folder.synced, boundary, use);
             if (folder.kind === 'sent')
                 sentPersisted = true;
         }
@@ -234,12 +250,33 @@ async function commitBatch(db, account, folder, uidValidity, fetched, direction,
     return db.transaction(async (client) => {
         const pending = [];
         for (const message of fetched) {
+            /*
+             * Back in the inbox: a message buddi holds as gone, placed outside
+             * the inbox (archived, labelled, trashed in another app), or put back
+             * here with its new uid unknown, arriving under a new uid is the same
+             * message — its row follows, and it is neither stored twice nor
+             * triaged again.
+             */
+            if (direction === 'in' && message.messageId) {
+                const back = await client.query(`update email.messages m
+              set folder_id = $3, uidvalidity = $4, uid = $5, gone_at = null, flags = $6::jsonb
+             from email.folders f
+            where f.id = m.folder_id and m.account_id = $1 and m.message_id = $2 and m.direction = 'in'
+              and (m.gone_at is not null or f.kind <> 'inbox' or m.uidvalidity = 0)
+              and not exists (select 1 from email.messages o
+                               where o.account_id = $1 and o.folder_id = $3 and o.uidvalidity = $4 and o.uid = $5)
+           returning m.id`, [account.id, message.messageId, folder.id, uidValidity, message.uid, JSON.stringify(message.flags)]);
+                if ((back.rowCount ?? 0) > 0) {
+                    await client.query(`update email.threads set last_folder_id = $2, last_uidvalidity = $3, last_uid = $4 where last_message_id = $1`, [String(back.rows[0].id), folder.id, uidValidity, message.uid]);
+                    continue;
+                }
+            }
             const { rows } = await client.query(`insert into email.messages
            (account_id, folder_id, uidvalidity, uid, message_id, thread_key, list_id, from_addr,
             to_addrs, cc, subject, date, internal_date, snippet, body_text, has_attachments,
-            attachments, flags, direction, triage_enqueued_at)
+            attachments, flags, direction, triage_enqueued_at, bulk)
          values ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10::jsonb, $11, $12, $13, $14, $15,
-                 $16, $17::jsonb, $18::jsonb, $19, $20)
+                 $16, $17::jsonb, $18::jsonb, $19, $20, $21)
          on conflict (account_id, folder_id, uidvalidity, uid) do nothing
          returning id`, [
                 account.id,
@@ -263,6 +300,7 @@ async function commitBatch(db, account, folder, uidValidity, fetched, direction,
                 direction,
                 // Sent mail is nobody's to triage; it is stamped as it lands.
                 direction === 'out' ? now : null,
+                message.bulk === true,
             ]);
             const id = rows[0]?.id;
             if (!id)
@@ -344,7 +382,276 @@ export function createInboxPollSource(opts) {
     const onlyFolder = opts.mailbox;
     const limit = Math.min(Math.max(1, opts.limit ?? MAX_PER_POLL), MAX_PER_POLL);
     const agentId = opts.agentId ?? TRIAGE_AGENT_ID;
-    return {
+    const slowPollMs = (opts.slowPollSeconds ?? IDLE_SLOW_POLL_SECONDS) * 1000;
+    /*
+     * One poll of one account at a time, whoever asked: the scheduled pass, or
+     * IDLE. `inFlight` is the lock; `again` is a change IDLE saw while a poll
+     * was already running, which earns exactly one more poll after it.
+     * `lastPolled` is this process's own clock of when each account last
+     * finished, for the slower safety-net period while IDLE is live.
+     */
+    const inFlight = new Map();
+    const again = new Set();
+    const lastPolled = new Map();
+    let watchers = null;
+    let watchCtx = null;
+    /** Run `work` as this account's poll. Callers check `inFlight` first; this sets it synchronously. */
+    function runLocked(accountId, work) {
+        const running = (async () => {
+            try {
+                await work();
+            }
+            finally {
+                inFlight.delete(accountId);
+                lastPolled.set(accountId, Date.now());
+                if (again.delete(accountId)) {
+                    const ctx = watcherCtx();
+                    // Like IDLE's own debounced poll: a failure is logged (a refused
+                    // login was already recorded by the poll), never left unhandled.
+                    if (ctx !== null) {
+                        pollNow(accountId).catch((err) => {
+                            (ctx.buddi?.log ?? ((line) => console.error(line)))(`email.inbox-poll: follow-up poll failed: ${err instanceof Error ? err.message : String(err)}`);
+                        });
+                    }
+                }
+            }
+        })();
+        inFlight.set(accountId, running);
+        return running;
+    }
+    function watcherCtx() {
+        return watchers ? watchCtx : null;
+    }
+    /** IDLE saw a change: this account's poll, now, or once more after the one running. */
+    function pollNow(accountId) {
+        const ctx = watcherCtx();
+        if (ctx === null)
+            return Promise.resolve();
+        if (inFlight.has(accountId)) {
+            again.add(accountId);
+            return Promise.resolve();
+        }
+        return runLocked(accountId, async () => {
+            const account = await findAccount(ctx.buddi.db, accountId);
+            if (account === null)
+                return;
+            const pass = { failures: [], triageReady: ctx.buddi.owner.hasAgent(agentId), saidWaiting: false };
+            await pollAccount(ctx, account, pass);
+            if (pass.failures.length > 0)
+                throw pass.failures[0];
+        });
+    }
+    /** One account's whole pass: connect, folders, flags, drain, on-arrival actions. */
+    async function pollAccount(ctx, account, pass) {
+        const log = ctx.buddi?.log ?? ((line) => console.error(line));
+        const env = opts.env ?? process.env;
+        const auth = await mailboxAuth(ctx, account, opts.env);
+        if (!auth.ok) {
+            // A typed configuration problem: say it once per poll and stop. An
+            // unattended run that needs a secret fails with a problem; it never
+            // hangs and never falls back to asking.
+            log(`email.inbox-poll: ${auth.problem.code}: ${auth.problem.message}`);
+            return;
+        }
+        const timeoutMs = Math.max(1, opts.timeoutMs ?? envInt(env, POLL_TIMEOUT_VAR, DEFAULT_POLL_TIMEOUT_MS));
+        const backfill = Math.max(0, opts.backfill ?? envInt(env, BACKFILL_VAR, DEFAULT_BACKFILL));
+        // The connection is opened for this poll and closed at the end of it,
+        // always. A long-lived IMAP session is a socket that silently dies while
+        // nobody is looking; one per poll is cheap (half a second) and honest.
+        // Both folders are walked over the same connection.
+        let client = null;
+        const pending = [];
+        /*
+         * Did this account's pass get all the way through? Only then is it a
+         * sync, and only then is `last_synced_at` moved: a Sent folder that
+         * timed out still let the inbox land, but it means what we hold for
+         * this mailbox is no longer everything the server has, and a metric
+         * that stamped it "as of now" would be saying otherwise.
+         */
+        let complete = true;
+        try {
+            try {
+                client = await withDeadline('connect', timeoutMs, opts.connect(account, auth.value), 
+                // We gave up waiting, but the connection may still arrive: close it
+                // rather than leave a socket nobody owns.
+                (late) => void late.close().catch(() => { }));
+            }
+            catch (err) {
+                // The provider refused the password it was given: written on the
+                // row so Settings → Email says "Password needed" (`logins.ts`).
+                if (isAuthFailure(err)) {
+                    await recordLoginFailure(ctx.buddi.db, account.id, err, ctx.buddi.clock.now()).catch(() => { });
+                    log(`email.inbox-poll: ${account.address} refused its stored password; Settings → Email → Set password`);
+                }
+                throw err;
+            }
+            await clearLoginFailure(ctx.buddi.db, account.id);
+            // Which folders this account has, discovered once and remembered.
+            // A caller that named one folder gets that folder and no listing:
+            // the CLI and the tests both drive a single mailbox on purpose.
+            const folders = onlyFolder
+                ? [await ensureFolder(ctx.buddi.db, account, onlyFolder, 'inbox', true)]
+                : (await discoverFolders(ctx.buddi.db, account, client, timeoutMs, log)).filter((f) => f.synced);
+            // New discovery persists Sent's LIST/STATUS boundary above. This
+            // loop is for an older or partially initialised row that still has
+            // no generation: try to plant it before INBOX work, but isolate a
+            // failed SELECT or cursor write so incoming mail still lands.
+            const attemptedSent = new Set();
+            for (const folder of folders.filter((f) => f.kind === 'sent' && f.uidValidity === null)) {
+                attemptedSent.add(folder.id);
+                try {
+                    await pollFolder(ctx, account, client, folder, { timeoutMs, backfill, limit, log });
+                }
+                catch (err) {
+                    log(`email.inbox-poll: could not initialize Sent folder ${account.address}/${folder.name}: ` +
+                        `${err instanceof Error ? err.message : String(err)}; continuing with INBOX`);
+                    complete = false;
+                    pass.failures.push(err);
+                }
+            }
+            // The inbox first: it is the one that wakes anybody, and a Sent
+            // folder that times out must not cost the new mail its run.
+            for (const folder of [...folders].filter((f) => !attemptedSent.has(f.id)).sort((a, b) => (a.kind === 'inbox' ? -1 : b.kind === 'inbox' ? 1 : 0))) {
+                try {
+                    const polled = await pollFolder(ctx, account, client, folder, {
+                        timeoutMs,
+                        backfill,
+                        limit,
+                        log,
+                    });
+                    pending.push(...polled.pending);
+                    if (folder.kind === 'inbox') {
+                        // After ingest, so the new rows are in the window with the
+                        // flags they arrived with. A re-sync that fails costs nobody
+                        // their mail: the rows landed, the runs are queued below, and
+                        // only this pass stops counting as a sync — the unread count
+                        // it would have corrected is not stamped as current.
+                        try {
+                            await syncFlags(ctx.buddi.db, account, client, polled.folder, polled.status, timeoutMs, log);
+                        }
+                        catch (err) {
+                            log(`email.inbox-poll: could not re-sync flags on ${account.address}/${folder.name}: ` +
+                                `${err instanceof Error ? err.message : String(err)}; new mail still landed`);
+                            complete = false;
+                            pass.failures.push(err);
+                        }
+                        // What left the inbox in another app (`presence.ts`): after the
+                        // flags, so a departed row keeps the flags it last had there.
+                        try {
+                            const reader = client;
+                            await checkPresence(ctx.buddi.db, account, reader, polled.folder, polled.status, {
+                                window: FLAG_SYNC_WINDOW,
+                                now: ctx.buddi.clock.now(),
+                                log,
+                                deadline: (op, work) => withDeadline(op, timeoutMs, work),
+                            });
+                        }
+                        catch (err) {
+                            log(`email.inbox-poll: could not check what left ${account.address}/${folder.name}: ` +
+                                `${err instanceof Error ? err.message : String(err)}; new mail still landed`);
+                            complete = false;
+                            pass.failures.push(err);
+                        }
+                        // Older mail never read for List-Unsubscribe / Precedence: a
+                        // bounded few, header lines only; a newsletter found keeps the
+                        // waiting cards for it the way a new one would.
+                        try {
+                            const found = await backfillBulk(ctx.buddi.db, client, polled.folder, polled.status, timeoutMs);
+                            if (found > 0) {
+                                log(`email.inbox-poll: ${found} older message(s) on ${account.address}/${folder.name} read as sent to many`);
+                                const proposals = ctx.buddi.proposals;
+                                if (proposals) {
+                                    const host = { db: ctx.buddi.db, owner: ctx.buddi.owner, proposals };
+                                    const now = ctx.buddi.clock.now();
+                                    if ((await sweepOpenBulkCards(host, now)) > 0)
+                                        await tellOwnerLearned(host, now);
+                                }
+                            }
+                        }
+                        catch (err) {
+                            log(`email.inbox-poll: could not re-read older mail's list headers on ${account.address}/${folder.name}: ` +
+                                `${err instanceof Error ? err.message : String(err)}; new mail still landed`);
+                        }
+                    }
+                }
+                catch (err) {
+                    if (folder.kind !== 'sent')
+                        throw err;
+                    log(`email.inbox-poll: could not poll Sent folder ${account.address}/${folder.name}: ` +
+                        `${err instanceof Error ? err.message : String(err)}; INBOX was still polled`);
+                    complete = false;
+                    pass.failures.push(err);
+                }
+            }
+            /*
+             * The pass finished. This is the one place `last_synced_at` moves,
+             * and it moves whether or not a single message was new: "we read
+             * this mailbox and there was nothing" is an answer, and without a
+             * mark of its own it is indistinguishable from never having looked.
+             */
+            if (complete)
+                await markAccountSynced(ctx.buddi.db, account.id, ctx.buddi.clock.now());
+        }
+        catch (err) {
+            complete = false;
+            if (err instanceof ImapTimeoutError) {
+                // A recorded give-up. Rethrowing it at the end of the pass is
+                // deliberate: `runSources` writes it to
+                // core.source_runs.last_error and emits `source.polled` with it,
+                // so a mailbox that stopped answering is visible rather than silent.
+                log(`email.inbox-poll: ${err.message} on ${account.address}; ` +
+                    `connection closed, retrying next poll`);
+            }
+            pass.failures.push(err);
+            // Whatever did land before the failure is still drained below: a
+            // Sent folder that timed out is no reason to sit on the inbox.
+        }
+        finally {
+            await client?.close().catch(() => { });
+        }
+        try {
+            const waiting = await drain(ctx, account, agentId, limit, pending, pass.triageReady);
+            if (waiting > 0 && !pass.saidWaiting) {
+                pass.saidWaiting = true;
+                log(`email.inbox-poll: no triage agent yet — accept the Mail offer on the dashboard`);
+            }
+            await recordTriageWaiting(ctx.buddi.db, account.id, pass.triageReady ? false : waiting > 0 ? true : null, ctx.buddi.clock.now());
+        }
+        catch (err) {
+            pass.failures.push(err);
+        }
+        /*
+         * Rules with an on-arrival mailbox action (archive, mark read, move):
+         * after the gate and the queue, over a connection of their own, and
+         * on the undo trail (`mailbox/arrival.ts`). Whatever is owed — this
+         * drain's, and an earlier poll's that failed — is tried; a failure is
+         * logged, counted, and costs nobody their triage.
+         */
+        /*
+         * Intent first: a mailbox change this process did not finish (buddi
+         * stopped between the server change and its record) left a `pending`
+         * trail row. Settle it with the server before any new change acts.
+         */
+        try {
+            const settled = await reconcilePendingChanges(ctx, account, { connect: opts.connect, ...(opts.env ? { env: opts.env } : {}) });
+            if (settled > 0)
+                log(`email.inbox-poll: settled ${settled} interrupted mailbox change(s) on ${account.address} with the server`);
+        }
+        catch (err) {
+            log(`email.inbox-poll: could not settle interrupted mailbox changes on ${account.address}: ` +
+                `${err instanceof Error ? err.message : String(err)}; the next poll tries again`);
+        }
+        {
+            try {
+                await applyPendingArrivals(ctx, account, { connect: opts.connect, ...(opts.env ? { env: opts.env } : {}) });
+            }
+            catch (err) {
+                log(`email.inbox-poll: on-arrival rule actions on ${account.address} failed: ` +
+                    `${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    }
+    const source = {
         id: 'email.inbox-poll',
         description: onlyFolder
             ? `Poll ${onlyFolder} for new mail and start a triage run for each new message.`
@@ -352,7 +659,6 @@ export function createInboxPollSource(opts) {
         every: POLL_EVERY_SECONDS,
         async poll(ctx) {
             const log = ctx.buddi?.log ?? ((line) => console.error(line));
-            const env = opts.env ?? process.env;
             // Accounts are plural (docs/email.md §2). Every enabled one is polled in
             // this pass, each with its own folders and its own cursor per folder. No
             // mailbox configured is still a valid, running state rather than a
@@ -362,7 +668,6 @@ export function createInboxPollSource(opts) {
             // a mailbox that stopped answering is collected and rethrown after every
             // other account has had its turn, so `runSources` still records it in
             // `core.source_runs.last_error` and the rest of the mail still lands.
-            const failures = [];
             /*
              * Is there anybody to hand new mail to? The triage agent is proposed by
              * this plugin, not shipped: until the owner accepts it, a run enqueued
@@ -371,140 +676,56 @@ export function createInboxPollSource(opts) {
              * runs wait, the messages stay unstamped, and the next poll after the
              * agent exists starts them. Said once per poll, not once per message.
              */
-            const triageReady = ctx.buddi.owner.hasAgent(agentId);
-            let saidWaiting = false;
+            const pass = { failures: [], triageReady: ctx.buddi.owner.hasAgent(agentId), saidWaiting: false };
+            // The IDLE watchers follow the accounts: one added, removed or turned
+            // off since the last pass is picked up here at the latest.
+            if (watchers)
+                await watchers.reconcile();
             for (const account of await listAccounts(ctx.buddi.db)) {
-                const auth = await mailboxAuth(ctx, account, opts.env);
-                if (!auth.ok) {
-                    // A typed configuration problem: say it once per poll and stop. An
-                    // unattended run that needs a secret fails with a problem; it never
-                    // hangs and never falls back to asking.
-                    log(`email.inbox-poll: ${auth.problem.code}: ${auth.problem.message}`);
+                // Never two polls of one account at once: IDLE already has it.
+                if (inFlight.has(account.id)) {
+                    log(`email.inbox-poll: ${account.address} is being polled already (IDLE); skipping it this pass`);
                     continue;
                 }
-                const timeoutMs = Math.max(1, opts.timeoutMs ?? envInt(env, POLL_TIMEOUT_VAR, DEFAULT_POLL_TIMEOUT_MS));
-                const backfill = Math.max(0, opts.backfill ?? envInt(env, BACKFILL_VAR, DEFAULT_BACKFILL));
-                // The connection is opened for this poll and closed at the end of it,
-                // always. A long-lived IMAP session is a socket that silently dies while
-                // nobody is looking; one per poll is cheap (half a second) and honest.
-                // Both folders are walked over the same connection.
-                let client = null;
-                const pending = [];
-                /*
-                 * Did this account's pass get all the way through? Only then is it a
-                 * sync, and only then is `last_synced_at` moved: a Sent folder that
-                 * timed out still let the inbox land, but it means what we hold for
-                 * this mailbox is no longer everything the server has, and a metric
-                 * that stamped it "as of now" would be saying otherwise.
-                 */
-                let complete = true;
-                try {
-                    client = await withDeadline('connect', timeoutMs, opts.connect(account, auth.value), 
-                    // We gave up waiting, but the connection may still arrive: close it
-                    // rather than leave a socket nobody owns.
-                    (late) => void late.close().catch(() => { }));
-                    // Which folders this account has, discovered once and remembered.
-                    // A caller that named one folder gets that folder and no listing:
-                    // the CLI and the tests both drive a single mailbox on purpose.
-                    const folders = onlyFolder
-                        ? [await ensureFolder(ctx.buddi.db, account, onlyFolder, 'inbox', true)]
-                        : (await discoverFolders(ctx.buddi.db, account, client, timeoutMs, log)).filter((f) => f.synced);
-                    // New discovery persists Sent's LIST/STATUS boundary above. This
-                    // loop is for an older or partially initialised row that still has
-                    // no generation: try to plant it before INBOX work, but isolate a
-                    // failed SELECT or cursor write so incoming mail still lands.
-                    const attemptedSent = new Set();
-                    for (const folder of folders.filter((f) => f.kind === 'sent' && f.uidValidity === null)) {
-                        attemptedSent.add(folder.id);
-                        try {
-                            await pollFolder(ctx, account, client, folder, { timeoutMs, backfill, limit, log });
-                        }
-                        catch (err) {
-                            log(`email.inbox-poll: could not initialize Sent folder ${account.address}/${folder.name}: ` +
-                                `${err instanceof Error ? err.message : String(err)}; continuing with INBOX`);
-                            complete = false;
-                            failures.push(err);
-                        }
-                    }
-                    // The inbox first: it is the one that wakes anybody, and a Sent
-                    // folder that times out must not cost the new mail its run.
-                    for (const folder of [...folders].filter((f) => !attemptedSent.has(f.id)).sort((a, b) => (a.kind === 'inbox' ? -1 : b.kind === 'inbox' ? 1 : 0))) {
-                        try {
-                            const polled = await pollFolder(ctx, account, client, folder, {
-                                timeoutMs,
-                                backfill,
-                                limit,
-                                log,
-                            });
-                            pending.push(...polled.pending);
-                            if (folder.kind === 'inbox') {
-                                // After ingest, so the new rows are in the window with the
-                                // flags they arrived with. A re-sync that fails costs nobody
-                                // their mail: the rows landed, the runs are queued below, and
-                                // only this pass stops counting as a sync — the unread count
-                                // it would have corrected is not stamped as current.
-                                try {
-                                    await syncFlags(ctx.buddi.db, account, client, polled.folder, polled.status, timeoutMs, log);
-                                }
-                                catch (err) {
-                                    log(`email.inbox-poll: could not re-sync flags on ${account.address}/${folder.name}: ` +
-                                        `${err instanceof Error ? err.message : String(err)}; new mail still landed`);
-                                    complete = false;
-                                    failures.push(err);
-                                }
-                            }
-                        }
-                        catch (err) {
-                            if (folder.kind !== 'sent')
-                                throw err;
-                            log(`email.inbox-poll: could not poll Sent folder ${account.address}/${folder.name}: ` +
-                                `${err instanceof Error ? err.message : String(err)}; INBOX was still polled`);
-                            complete = false;
-                            failures.push(err);
-                        }
-                    }
-                    /*
-                     * The pass finished. This is the one place `last_synced_at` moves,
-                     * and it moves whether or not a single message was new: "we read
-                     * this mailbox and there was nothing" is an answer, and without a
-                     * mark of its own it is indistinguishable from never having looked.
-                     */
-                    if (complete)
-                        await markAccountSynced(ctx.buddi.db, account.id, ctx.buddi.clock.now());
-                }
-                catch (err) {
-                    complete = false;
-                    if (err instanceof ImapTimeoutError) {
-                        // A recorded give-up. Rethrowing it at the end of the pass is
-                        // deliberate: `runSources` writes it to
-                        // core.source_runs.last_error and emits `source.polled` with it,
-                        // so a mailbox that stopped answering is visible rather than silent.
-                        log(`email.inbox-poll: ${err.message} on ${account.address}; ` +
-                            `connection closed, retrying next poll`);
-                    }
-                    failures.push(err);
-                    // Whatever did land before the failure is still drained below: a
-                    // Sent folder that timed out is no reason to sit on the inbox.
-                }
-                finally {
-                    await client?.close().catch(() => { });
-                }
-                try {
-                    const waiting = await drain(ctx, account, agentId, limit, pending, triageReady);
-                    if (waiting > 0 && !saidWaiting) {
-                        saidWaiting = true;
-                        log(`email.inbox-poll: no triage agent yet — accept the Mail offer on the dashboard`);
-                    }
-                    await recordTriageWaiting(ctx.buddi.db, account.id, triageReady ? false : waiting > 0 ? true : null, ctx.buddi.clock.now());
-                }
-                catch (err) {
-                    failures.push(err);
-                }
+                // IDLE is live: the server says when mail comes, and this pass is
+                // the safety net, at the slower period.
+                const last = lastPolled.get(account.id);
+                if (watchers?.state(account.id) === 'live' && last !== undefined && Date.now() - last < slowPollMs)
+                    continue;
+                await runLocked(account.id, () => pollAccount(ctx, account, pass));
             }
-            if (failures.length > 0)
-                throw failures[0];
+            if (pass.failures.length > 0)
+                throw pass.failures[0];
         },
     };
+    if (opts.idle) {
+        const idle = opts.idle;
+        source.watch = (ctx) => {
+            const manager = new IdleWatchers({
+                idle,
+                host: ctx.buddi,
+                poll: (accountId) => pollNow(accountId),
+                ...(opts.env ? { env: opts.env } : {}),
+                ...(opts.idleTuning ?? {}),
+            });
+            watchers = manager;
+            watchCtx = ctx;
+            void manager.reconcile();
+            return {
+                async stop() {
+                    if (watchers === manager) {
+                        watchers = null;
+                        watchCtx = null;
+                    }
+                    await manager.stop();
+                    // A poll IDLE started finishes (each IMAP call has its deadline);
+                    // nothing of this watcher outlives `stop`.
+                    await Promise.allSettled([...inFlight.values()]);
+                },
+            };
+        };
+    }
+    return source;
 }
 /**
  * One folder of one account: open it, honour its generation, fetch what is new,
@@ -586,15 +807,14 @@ async function pollFolder(ctx, account, client, folder, opts) {
  * than skipped.
  *
  * **A message gone from INBOX** (archived or deleted in another client) is
- * simply not in the answer. `email.messages` has no "still in the inbox"
- * field, and this does not invent one: such a row keeps the flags it last had.
- * The full fetch could tell it apart; CONDSTORE without QRESYNC cannot.
+ * simply not in the answer here, and keeps the flags it last had; the
+ * presence check after this (`presence.ts`) is what records that it left.
  */
 async function syncFlags(db, account, client, folder, status, timeoutMs, log) {
     const started = Date.now();
     const serverModseq = status.highestModseq ?? null;
     const { rows } = await db.query(`select uid from email.messages
-      where folder_id = $1 and uidvalidity = $2
+      where folder_id = $1 and uidvalidity = $2 and gone_at is null
       order by uid desc
       limit $3`, [folder.id, status.uidValidity, FLAG_SYNC_WINDOW]);
     const uids = rows.map((r) => Number(r.uid)).sort((a, b) => a - b);
@@ -632,11 +852,39 @@ async function syncFlags(db, account, client, folder, status, timeoutMs, log) {
     log(`email.inbox-poll: flags on ${account.address}/${folder.name} re-synced via ${mode} in ` +
         `${Date.now() - started}ms — ${uids.length} held, ${reported} reported, ${updated} changed`);
 }
+/**
+ * Read `List-Unsubscribe` and `Precedence` for held messages stored before
+ * `messages.bulk` was read at ingest (`bulk_checked = false`), at most
+ * `BULK_BACKFILL_PER_POLL` a poll, newest first, and mark them. A message the
+ * server no longer has is marked too: there is nothing more to read. Returns
+ * how many were found to be bulk. A client without `fetchBulk` reads nothing.
+ */
+export async function backfillBulk(db, client, folder, status, timeoutMs, limit = BULK_BACKFILL_PER_POLL) {
+    if (!client.fetchBulk)
+        return 0;
+    const { rows } = await db.query(`select uid from email.messages
+      where folder_id = $1 and uidvalidity = $2 and not bulk_checked and gone_at is null
+      order by uid desc
+      limit $3`, [folder.id, status.uidValidity, limit]);
+    const uids = rows.map((r) => Number(r.uid));
+    if (uids.length === 0)
+        return 0;
+    const read = await withDeadline('bulk headers', timeoutMs, client.fetchBulk(folder.name, uids));
+    const bulk = uids.filter((uid) => read.get(uid) === true);
+    await db.query(`update email.messages
+        set bulk_checked = true, bulk = bulk or uid = any($4::bigint[])
+      where folder_id = $1 and uidvalidity = $2 and uid = any($3::bigint[])`, [folder.id, status.uidValidity, uids, bulk]);
+    return bulk.length;
+}
 /** Persist the generation and the cursor together, and return the fresh row. */
 async function plantCursor(db, folderId, uidValidity, lastUid) {
     const { rows } = await db.query(`update email.folders set uidvalidity = $2, last_uid = $3,
             -- A modseq belongs to its generation; the next re-sync starts over.
-            highest_modseq = case when uidvalidity is distinct from $2 then null else highest_modseq end
+            highest_modseq = case when uidvalidity is distinct from $2 then null else highest_modseq end,
+            -- So does what the presence check last saw (presence.ts).
+            presence_uidnext = case when uidvalidity is distinct from $2 then null else presence_uidnext end,
+            presence_exists = case when uidvalidity is distinct from $2 then null else presence_exists end,
+            presence_modseq = case when uidvalidity is distinct from $2 then null else presence_modseq end
       where id = $1 returning ${FOLDER_COLUMNS}`, [folderId, uidValidity, lastUid]);
     const row = rows[0];
     if (!row)
@@ -751,6 +999,11 @@ async function drain(ctx, account, agentId, limit, pending, triageReady = true)
                     detail: decision.detail,
                     status: 'done',
                 }, ctx.buddi.clock.now());
+                // The rule's mailbox action is owed before the stamp lands: a
+                // failed writer connection later must not lose it.
+                if (decision.policy?.params.onArrival) {
+                    await oweArrivalAction(tx, account.id, { messageId: message.id, policy: decision.policy });
+                }
                 await stampOn(tx, message.id, ctx.buddi.clock.now());
             });
             (ctx.buddi?.log ?? (() => { }))(`email.inbox-poll: ${message.from} handled by policy ${decision.policy.scope} ` +
@@ -780,6 +1033,11 @@ async function drain(ctx, account, agentId, limit, pending, triageReady = true)
             throw err;
         }
         await settleEvent(ctx.buddi.db, message.id, 'done');
+        // Owed before the stamp: once stamped the message is never drained
+        // again, so the action has to be on record by then (`arrival.ts`).
+        if (!decision.refused && decision.policy?.params.onArrival) {
+            await oweArrivalAction(ctx.buddi.db, account.id, { messageId: message.id, policy: decision.policy });
+        }
         await stamp(ctx, message.id);
     }
     return waiting;
@@ -830,7 +1088,9 @@ async function promptFor(ctx, account, message, decision, policies) {
     const senderPolicy = decision.policy ??
         policies.find((p) => p.scope === 'sender' && p.matcher === message.from.trim().toLowerCase()) ??
         null;
+    const place = await placeOf(ctx.buddi.db, message.id);
     return triagePrompt({
+        ...(place ? { place } : {}),
         messageId: message.id,
         from: message.from,
         to: message.to,
@@ -861,6 +1121,21 @@ async function promptFor(ctx, account, message, decision, policies) {
         ...(instructionFor(decision) ? { instruction: instructionFor(decision) } : {}),
     });
 }
+/** Where a message is now, in words, when that is not the inbox (`placeWords`); '' otherwise. */
+async function placeOf(db, messageId) {
+    const { rows } = await db.query(`select m.gone_at, m.direction, f.kind, f.name, f.special_use
+       from email.messages m left join email.folders f on f.id = m.folder_id where m.id = $1`, [messageId]);
+    const r = rows[0];
+    if (!r)
+        return '';
+    return placeWords({
+        kind: r.kind ?? null,
+        name: r.name ?? null,
+        specialUse: r.special_use ?? null,
+        gone: r.gone_at !== null && r.gone_at !== undefined,
+        direction: r.direction === 'out' ? 'out' : 'in',
+    });
+}
 /** How many earlier turns of the thread are quoted, and how many are a line. */
 export const THREAD_TURNS_QUOTED = 3;
 export const THREAD_TURNS_LISTED = 10;