@cosmicdrift/kumiko-samples 0.392.2 → 0.394.0

package/packages/bundled-features/src/inbound-mail-foundation/watch-supervisor.ts

@@ -43,45 +43,42 @@
 // a Redis outage or expired TTL) tears down the local watcher instead of
 // racing a new holder.
 
-import {
-  configuredPiiSubjectKms,
-  decryptPiiFieldValues,
-} from "@cosmicdrift/kumiko-framework/crypto";
-import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
-import { fetchOne, insertOne, selectMany, updateMany } from "@cosmicdrift/kumiko-framework/db";
 import { SYSTEM_USER_ID } from "@cosmicdrift/kumiko-framework/engine";
 import {
   systemScopedRunner,
   tenantScopedSecrets,
 } from "@cosmicdrift/kumiko-framework/internal/row-level-security";
-import type { DistributedLock } from "@cosmicdrift/kumiko-framework/pipeline";
-import { Temporal } from "@cosmicdrift/kumiko-types/temporal";
-import { isPlainObject } from "@cosmicdrift/kumiko-types/type-guards";
+import { mapWithConcurrency } from "../shared/index.ts";
+import { createAccountHealth } from "./account-health.ts";
+import { createAccountPoller } from "./account-poll.ts";
+import { pollScheduleOf, withPollDeadline } from "./account-poll-schedule.ts";
+import { registerMailAccountWakeListener } from "./account-wake.ts";
 import { InboundMailAccountStatuses, InboundMailFoundationHandlers } from "./constants.ts";
-import { MAIL_ACCOUNT_PII_FIELDS, seenMessageTable, syncCursorTable } from "./entities.ts";
-import {
-  createOAuthAccessTokenManager,
-  type OAuthAccessTokenManager,
-  usesFoundationManagedOAuth,
-} from "./oauth-access-token.ts";
-import { mailAccountsProjectionTable } from "./projection.ts";
-import { resolveInboundProviderForAccount } from "./provider-factory.ts";
+import { describeError } from "./describe-error.ts";
+import { currentCountBucket } from "./ingest-failures.ts";
+import { createOAuthAccessTokenManager, usesFoundationManagedOAuth } from "./oauth-access-token.ts";
 import {
-  type InboundMailContext,
-  type InboundMailProviderPlugin,
   isInboundAuthError,
   isInboundCursorInvalidError,
   isInboundRateLimitError,
-  type MailAccountRecord,
-  type RawInboundMessage,
-  type SyncCursorPayload,
-} from "./types.ts";
-
-// Provider errors can embed token or response data in `message`; logs carry
-// only the error class.
-function describeError(err: unknown): string {
-  return err instanceof Error ? err.name : typeof err;
-}
+} from "./provider-errors.ts";
+import { resolveInboundProviderForAccount } from "./provider-factory.ts";
+import { createMessageIngester } from "./supervisor-ingest.ts";
+import {
+  createSyncAccountNow,
+  type InFlightSyncs,
+  runExclusivePerAccount,
+} from "./sync-account-now.ts";
+import { createSyncCursorStore } from "./sync-cursor-store.ts";
+import { listSyncableAccounts, type SyncableAccount } from "./syncable-accounts.ts";
+import type { InboundMailContext, InboundMailProviderPlugin, MailAccountRecord } from "./types.ts";
+import { createWatchCatchUp } from "./watch-catch-up.ts";
+import { createWatchLease } from "./watch-lease.ts";
+import type {
+  InboundMailSupervisor,
+  InboundMailSupervisorDeps,
+  WatcherState,
+} from "./watch-supervisor-types.ts";
 
 const DEFAULT_POLL_INTERVAL_MS = 5 * 60 * 1000;
 const DEFAULT_BACKFILL_WINDOW_DAYS = 30;
@@ -89,92 +86,6 @@ const DEFAULT_MAX_MESSAGES_PER_POLL = 200;
 const DEFAULT_WATCH_BACKOFF_INITIAL_MS = 5_000;
 const DEFAULT_WATCH_BACKOFF_MAX_MS = 5 * 60 * 1000;
 const DEFAULT_WATCH_LEASE_TTL_SECONDS = 90;
-/** One cursor per account (one mailbox inbox) for now; multi-folder support
- *  can add further scopes later without a schema change. */
-const CURSOR_SCOPE = "default";
-
-const SEEN_LOOKUP_CHUNK_SIZE = 1000;
-
-export type InboundMailSupervisorDeps = {
-  /** Slim context for provider calls (`registry` is required; `secrets` is
-   *  used for provider credential reads). */
-  readonly providerCtx: InboundMailContext;
-  /** App database — direct reads from read_mail_accounts (account snapshot)
-   *  and direct writes to store_mail_sync_cursors (unmanaged store). */
-  readonly db: DbConnection;
-  /** Standard dispatcher with SystemUser — carries ingest-message and
-   *  update-account. */
-  readonly dispatchWrite: (args: {
-    readonly handlerQn: string;
-    readonly payload: unknown;
-    readonly tenantId: string;
-  }) => Promise<{
-    readonly isSuccess: boolean;
-    readonly data?: unknown;
-    readonly error?: unknown;
-  }>;
-  /** Persists the raw MIME body (file-foundation) and returns its bodyRef
-   *  for the event. Without this hook, runs in snippet-only mode
-   *  (bodyRef = ""). */
-  readonly storeBody?: (account: MailAccountRecord, msg: RawInboundMessage) => Promise<string>;
-  readonly pollIntervalMs?: number;
-  readonly backfillWindowDays?: number;
-  readonly maxMessagesPerPoll?: number;
-  readonly watchBackoffInitialMs?: number;
-  readonly watchBackoffMaxMs?: number;
-  /** Coordinates the IMAP watch (not the poll) across worker processes:
-   *  before `plugin.watch()`, ensureWatcher claims a TTL lease for the
-   *  account via `lock.acquire` and renews it via `lock.renew` while the
-   *  watcher lives. Omit for single-process deployments — every active
-   *  account is watched locally, same as before. */
-  readonly lock?: DistributedLock;
-  /** Access-token lifecycle for OAuth accounts. Defaults to a manager built
-   *  from `db` + `providerCtx.secrets`; pass the instance shared with
-   *  `createInboundMailConnectRoutes` so a fresh connect primes this cache. */
-  readonly oauthTokens?: OAuthAccessTokenManager;
-  /** TTL (seconds) for the per-account watch lease. Default 90s, renewed
-   *  every ttl/3. Only used when `lock` is set. */
-  readonly watchLeaseTtlSeconds?: number;
-  readonly log?: (line: string) => void;
-};
-
-type WatcherState = {
-  stop: (() => Promise<void>) | null;
-  backoffMs: number;
-  restartTimer: ReturnType<typeof setTimeout> | null;
-  /** Incremented by stop() to prevent a delayed restart from reviving
-   *  an already stopped watcher. */
-  generation: number;
-  /** Token from `deps.lock.acquire` while this worker owns the account's
-   *  watch lease; null when unclaimed (no `deps.lock`, or claim lost). */
-  lockToken: string | null;
-  renewTimer: ReturnType<typeof setTimeout> | null;
-  /** Consecutive renew() throws — tear down once failures span a full TTL. */
-  renewFailures: number;
-  /** True while plugin.watch() is in flight (re-entrancy guard). */
-  starting: boolean;
-};
-
-type WatchLeaseResult = "acquired" | "no-token" | "stale";
-
-export type InboundMailSupervisor = {
-  readonly start: () => Promise<void>;
-  /** Reconciliation run across all active accounts; also usable standalone
-   *  for tests and manual operational triggers. */
-  readonly pollOnce: () => Promise<void>;
-  readonly stop: () => Promise<void>;
-};
-
-type ActiveMailAccountRow = {
-  readonly id: string;
-  readonly tenantId: string;
-  readonly provider: string;
-  readonly authMethod: string;
-  readonly ownerUserId?: string | null;
-  readonly address: string;
-  readonly displayName?: string;
-  readonly watchState?: string;
-};
 
 export function createInboundMailSupervisor(
   deps: InboundMailSupervisorDeps,
@@ -182,6 +93,7 @@ export function createInboundMailSupervisor(
   const pollIntervalMs = deps.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
   const backfillWindowDays = deps.backfillWindowDays ?? DEFAULT_BACKFILL_WINDOW_DAYS;
   const maxMessagesPerPoll = deps.maxMessagesPerPoll ?? DEFAULT_MAX_MESSAGES_PER_POLL;
+  const { pollConcurrency, accountPollDeadlineMs } = pollScheduleOf(deps);
   const backoffInitialMs = deps.watchBackoffInitialMs ?? DEFAULT_WATCH_BACKOFF_INITIAL_MS;
   const backoffMaxMs = deps.watchBackoffMaxMs ?? DEFAULT_WATCH_BACKOFF_MAX_MS;
   const leaseTtlSeconds = deps.watchLeaseTtlSeconds ?? DEFAULT_WATCH_LEASE_TTL_SECONDS;
@@ -229,156 +141,18 @@ export function createInboundMailSupervisor(
   let running = false;
   let pollTimer: ReturnType<typeof setTimeout> | null = null;
   let pollInFlight: Promise<void> | null = null;
+  let unregisterWake: (() => void) | null = null;
   const watchers = new Map<string, WatcherState>();
 
-  // ---------------------------------------------------------------
-  // Account snapshot: active accounts of all tenants with the address
-  // decrypted (providers never work with ciphertext).
-  // ---------------------------------------------------------------
-  async function listActiveAccounts(): Promise<readonly MailAccountRecord[]> {
-    const rows = await selectMany<ActiveMailAccountRow>(systemDb, mailAccountsProjectionTable, {
-      status: InboundMailAccountStatuses.active,
-    });
-    const piiKms = configuredPiiSubjectKms();
-    const records: MailAccountRecord[] = [];
-    for (const row of rows) {
-      let decrypted: Record<string, unknown>;
-      try {
-        decrypted = piiKms
-          ? await decryptPiiFieldValues(row, MAIL_ACCOUNT_PII_FIELDS, piiKms, {
-              requestId: "inbound-mail-foundation:supervisor:list-accounts",
-            })
-          : row;
-      } catch (error) {
-        // Class name only: the message may carry ciphertext or key details.
-        const errorName = error instanceof Error ? error.name : "unknown error";
-        log(`inbound-mail: account ${row.id} skipped — address did not decrypt (${errorName})`);
-        continue;
-      }
-      const decryptedAddress = decrypted["address"];
-      // Falling back to row.address would hand the ciphertext to a provider.
-      // One undecryptable account must not stop polling of every other tenant's accounts.
-      if (typeof decryptedAddress !== "string") {
-        log(`inbound-mail: account ${row.id} skipped — address did not decrypt to text`);
-        continue;
-      }
-      records.push({
-        id: row.id,
-        tenantId: row.tenantId,
-        provider: row.provider,
-        authMethod: row.authMethod,
-        ownerUserId: row.ownerUserId ?? null,
-        address: decryptedAddress,
-        displayName: row.displayName ?? "",
-        status: InboundMailAccountStatuses.active,
-        watchState: row.watchState ?? "idle",
-      });
-    }
-    return records;
-  }
-
-  // ---------------------------------------------------------------
-  // Cursor-Persistenz (unmanaged direct-write store).
-  // ---------------------------------------------------------------
-  async function loadCursor(accountId: string): Promise<SyncCursorPayload | null> {
-    const row = await fetchOne<{ cursor: string }>(systemDb, syncCursorTable, {
-      accountId,
-      scope: CURSOR_SCOPE,
-    });
-    if (!row) return null;
-    try {
-      const parsed: unknown = JSON.parse(row.cursor);
-      return isPlainObject(parsed) ? parsed : null;
-    } catch {
-      return null; // a corrupt cursor counts as none → backfill; dedup catches duplicates
-    }
-  }
-
-  async function saveCursor(
-    account: Pick<MailAccountRecord, "id" | "tenantId">,
-    cursor: SyncCursorPayload,
-  ): Promise<void> {
-    const accountId = account.id;
-    const now = Temporal.Now.instant().toString();
-    const serialized = JSON.stringify(cursor);
-    const existing = await fetchOne<{ id: string }>(systemDb, syncCursorTable, {
-      accountId,
-      scope: CURSOR_SCOPE,
-    });
-    if (existing) {
-      await updateMany(
-        systemDb,
-        syncCursorTable,
-        { cursor: serialized, updatedAt: now },
-        { accountId, scope: CURSOR_SCOPE },
-      );
-      // skip: the update path is complete; do not fall through to the insert branch.
-      return;
-    }
-    await insertOne(systemDb, syncCursorTable, {
-      id: crypto.randomUUID(),
-      // Direct-write store: no executor stamps the NOT NULL tenant_id for us.
-      tenantId: account.tenantId,
-      accountId,
-      scope: CURSOR_SCOPE,
-      cursor: serialized,
-      updatedAt: now,
-    });
-  }
-
-  async function resetCursor(accountId: string): Promise<void> {
-    // No deleteMany import is needed: an empty cursor parses to null, so the
-    // next fetch runs as a backfill.
-    await updateMany(
-      systemDb,
-      syncCursorTable,
-      { cursor: "", updatedAt: Temporal.Now.instant().toString() },
-      { accountId, scope: CURSOR_SCOPE },
-    );
-  }
+  const cursors = createSyncCursorStore(systemDb);
 
-  // ---------------------------------------------------------------
-  // Ingest: every message goes through the standard write handler.
-  // ---------------------------------------------------------------
-  async function ingestBatch(
-    account: MailAccountRecord,
-    msgs: readonly RawInboundMessage[],
-    cursorSnapshot: string,
-  ): Promise<void> {
-    for (const msg of msgs) {
-      const bodyRef = deps.storeBody && msg.rawMime ? await deps.storeBody(account, msg) : "";
-      const result = await deps.dispatchWrite({
-        handlerQn: InboundMailFoundationHandlers.ingestMessage,
-        tenantId: account.tenantId,
-        payload: {
-          accountId: account.id,
-          ownerUserId: account.ownerUserId,
-          providerName: account.provider,
-          providerMessageId: msg.providerMessageId,
-          messageIdHeader: msg.messageIdHeader,
-          providerThreadId: msg.providerThreadId,
-          references: msg.references,
-          from: msg.from,
-          to: msg.to,
-          cc: msg.cc,
-          subject: msg.subject,
-          snippet: msg.snippet,
-          receivedAtIso: msg.receivedAtIso,
-          bodyRef,
-          scope: msg.scope,
-          providerCursor: cursorSnapshot,
-        },
-      });
-      if (!result.isSuccess) {
-        // A single-message failure aborts the batch without persisting the cursor.
-        // The next tick refetches from the old cursor, and deduplication skips
-        // messages that were already processed.
-        throw new Error(
-          `ingest-message failed for account ${account.id}: ${JSON.stringify(result.error ?? {})}`,
-        );
-      }
-    }
-  }
+  const ingester = createMessageIngester({
+    db: systemDb,
+    dispatchWrite: deps.dispatchWrite,
+    log,
+    ...(deps.storeBody && { storeBody: deps.storeBody }),
+  });
+  const health = createAccountHealth({ db: systemDb, dispatchWrite: deps.dispatchWrite, log });
 
   async function markAccount(
     account: MailAccountRecord,
@@ -413,7 +187,7 @@ export function createInboundMailSupervisor(
     }
     if (isInboundCursorInvalidError(err)) {
       log(`inbound-mail: account ${account.id} cursor invalid — full resync in backfill window`);
-      await resetCursor(account.id);
+      await cursors.reset(account.id);
       return true;
     }
     // Transient or unknown error: log it and retry on the next tick.
@@ -421,185 +195,117 @@ export function createInboundMailSupervisor(
     return true;
   }
 
-  // ---------------------------------------------------------------
-  // Poll (Reconciliation).
-  // ---------------------------------------------------------------
-  function unknownMessageIdsFilterFor(account: MailAccountRecord) {
-    return async (providerMessageIds: readonly string[]): Promise<ReadonlySet<string>> => {
-      if (providerMessageIds.length === 0) return new Set();
-      // One bind parameter per id: a full-mailbox listing would exceed Postgres' 65535 limit.
-      const seenIds = new Set<string>();
-      for (let start = 0; start < providerMessageIds.length; start += SEEN_LOOKUP_CHUNK_SIZE) {
-        const seen = await selectMany<{ providerMessageId: string }>(
-          systemDb,
-          seenMessageTable,
-          {
-            tenantId: account.tenantId,
-            accountId: account.id,
-            providerMessageId: {
-              in: providerMessageIds.slice(start, start + SEEN_LOOKUP_CHUNK_SIZE),
-            },
-          },
-          { fields: ["providerMessageId"] },
-        );
-        for (const row of seen) seenIds.add(row.providerMessageId);
-      }
-      return new Set(providerMessageIds.filter((id) => !seenIds.has(id)));
-    };
+  const { pollAccount } = createAccountPoller({
+    systemDb,
+    cursors,
+    ingester,
+    health,
+    providerCtx,
+    dispatchWrite: deps.dispatchWrite,
+    backfillWindowDays,
+    maxMessagesPerPoll,
+    resolveAccessToken,
+    handleSyncError,
+    log,
+  });
+
+  function resolvePluginOrLog(account: SyncableAccount): InboundMailProviderPlugin | undefined {
+    try {
+      return resolveInboundProviderForAccount(providerCtx, account);
+    } catch (err) {
+      log(`inbound-mail: ${describeError(err)}`);
+      return undefined;
+    }
   }
 
-  async function pollAccount(
-    account: MailAccountRecord,
+  const inFlightSyncs: InFlightSyncs = new Map();
+
+  const pollAccountWithDeadline = withPollDeadline(pollAccount, {
+    ms: accountPollDeadlineMs,
+    onDeadline: async (account) => {
+      log(
+        `inbound-mail: account ${account.id} sync passed its ${accountPollDeadlineMs}ms deadline`,
+      );
+      await health.onPollFailure(account, "transient");
+    },
+  });
+
+  async function pollAccountOfTick(
+    account: SyncableAccount,
     plugin: InboundMailProviderPlugin,
   ): Promise<void> {
-    let cursor = await loadCursor(account.id);
-    let budget = maxMessagesPerPoll;
-    try {
-      // hasMore loop: paginate within this tick until the message budget is
-      // exhausted.
-      for (;;) {
-        const accessToken = await resolveAccessToken(account, plugin);
-        const result = await plugin.fetch(providerCtx, account, cursor, {
-          backfillWindowDays,
-          maxMessages: budget,
-          filterUnknownMessageIds: unknownMessageIdsFilterFor(account),
-          ...(accessToken !== undefined && { accessToken }),
+    // Joining would hang this tick on the same stuck sync; wake and syncAccountNow still join.
+    if (inFlightSyncs.has(account.id)) {
+      log(`inbound-mail: account ${account.id} is still syncing, skipped this tick`);
+    } else {
+      await runExclusivePerAccount(inFlightSyncs, account.id, async () => {
+        await pollAccountWithDeadline(account, plugin, {
+          countBucket: currentCountBucket(deps.now, pollIntervalMs),
         });
-        await ingestBatch(account, result.messages, JSON.stringify(result.nextCursor));
-        await saveCursor(account, result.nextCursor);
-        cursor = result.nextCursor;
-        budget -= result.messages.length;
-        if (!result.hasMore || budget <= 0) break;
-      }
-    } catch (err) {
-      await handleSyncError(account, err);
+      });
     }
+    if (running) await ensureWatcher(account, plugin);
   }
 
   async function pollOnce(): Promise<void> {
-    const accounts = await listActiveAccounts();
-    for (const account of accounts) {
-      let plugin: InboundMailProviderPlugin;
+    const accounts = await listSyncableAccounts(systemDb, log, { health });
+    // One account failing must not abort the others, and mapWithConcurrency would reject on the first error.
+    await mapWithConcurrency(accounts, Math.max(1, pollConcurrency), async (account) => {
       try {
-        plugin = resolveInboundProviderForAccount(providerCtx, account);
+        const plugin = resolvePluginOrLog(account);
+        if (plugin) await pollAccountOfTick(account, plugin);
       } catch (err) {
-        log(`inbound-mail: ${describeError(err)}`);
-        continue;
+        log(`inbound-mail: account ${account.id} tick failed: ${describeError(err)}`);
       }
-      await pollAccount(account, plugin);
-      if (running) await ensureWatcher(account, plugin);
-    }
-    // Remove watchers for accounts that are no longer active.
-    const activeIds = new Set(accounts.map((a) => a.id));
+    });
+    // Remove watchers for accounts that are no longer syncable.
+    const syncableIds = new Set(accounts.map((a) => a.id));
     for (const accountId of watchers.keys()) {
-      if (!activeIds.has(accountId)) await stopWatcher(accountId);
+      if (!syncableIds.has(accountId)) await stopWatcher(accountId);
     }
   }
 
+  const watchCatchUp = createWatchCatchUp({
+    inFlight: inFlightSyncs,
+    pollAccount: pollAccountWithDeadline,
+    isRunning: () => running,
+    log,
+  });
+
+  const syncAccountNow = createSyncAccountNow({
+    inFlight: inFlightSyncs,
+    loadAccount: async (onlyAccountId) =>
+      (await listSyncableAccounts(systemDb, log, { onlyAccountId }))[0],
+    resolvePlugin: resolvePluginOrLog,
+    pollAccount: pollAccountWithDeadline,
+    ensureWatcher,
+    isRunning: () => running,
+  });
+
   // ---------------------------------------------------------------
   // Watch lifecycle with backoff restart.
   // ---------------------------------------------------------------
-  // Heartbeat for a held watch lease. Independent of the connection's own
-  // lifecycle — it keeps renewing across reconnect backoff, not just while
-  // `plugin.watch()` is actually connected, so a flaky IMAP link doesn't
-  // make this worker lose the account to a peer mid-backoff.
-  function scheduleRenew(
-    account: MailAccountRecord,
-    state: WatcherState,
-    generation: number,
-  ): void {
-    state.renewTimer = setTimeout(() => {
-      void renewLeaseOnce(account, state, generation);
-    }, renewIntervalMs);
-  }
-
-  async function tearDownAfterRenewFailures(
-    account: MailAccountRecord,
-    state: WatcherState,
-    reason: string,
-  ): Promise<void> {
-    log(
-      `inbound-mail: watch lease ${reason} for account ${account.id} — tearing down local watcher`,
-    );
-    state.lockToken = null;
-    await stopWatcher(account.id);
-  }
-
-  async function renewLeaseOnce(
-    account: MailAccountRecord,
-    state: WatcherState,
-    generation: number,
-  ): Promise<void> {
-    state.renewTimer = null;
-    // skip: supervisor stopped, generation superseded, or lease already lost — stale timer fire, nothing to renew.
-    if (!running || state.generation !== generation || !state.lockToken || !deps.lock) return;
-    let renewed: boolean;
-    try {
-      renewed = await deps.lock.renew(account.id, state.lockToken, leaseTtlSeconds);
-    } catch (err) {
-      log(
-        `inbound-mail: watch lease renew for account ${account.id} failed: ${describeError(err)}`,
-      );
-      state.renewFailures += 1;
-      // After failures spanning a full lease TTL, tear down — otherwise a
-      // peer can acquire the expired key while we still hold an IDLE conn.
-      if (state.renewFailures * renewIntervalMs >= leaseTtlSeconds * 1000) {
-        await tearDownAfterRenewFailures(account, state, "renew failed across TTL");
-        // skip: connection torn down, no reschedule
-        return;
-      }
-      rescheduleIfStillHeld(account, state, generation);
-      // skip: renew failed but TTL not yet exhausted, retry already scheduled
-      return;
-    }
-    state.renewFailures = 0;
-    if (!renewed) {
-      await tearDownAfterRenewFailures(account, state, "lost");
-      // skip: lease lost and torn down, nothing to reschedule
-      return;
-    }
-    rescheduleIfStillHeld(account, state, generation);
-  }
-
-  function rescheduleIfStillHeld(
-    account: MailAccountRecord,
-    state: WatcherState,
-    generation: number,
-  ): void {
-    if (running && state.generation === generation && state.lockToken) {
-      scheduleRenew(account, state, generation);
-    }
-  }
-
-  // kumiko-lint-ignore complexity-budget yield-point stale-map check after acquire is intentional
-  async function acquireWatchLease(
-    account: MailAccountRecord,
-    state: WatcherState,
-  ): Promise<WatchLeaseResult> {
-    if (!deps.lock || state.lockToken) return "acquired";
-    const token = await deps.lock.acquire(account.id, { ttlSeconds: leaseTtlSeconds });
-    if (!token) return "no-token";
-    // The acquire() await is a yield point: a concurrent stopWatcher()
-    // (e.g. from a lost-lease renewal on a different generation) may have
-    // already retired this exact state and removed it from the map. If so,
-    // committing the fresh token onto it would leak the lease — nothing
-    // would ever release it, blocking failover for the full TTL.
-    if (watchers.get(account.id) !== state) {
-      await deps.lock.release(account.id, token);
-      return "stale";
-    }
-    state.lockToken = token;
-    scheduleRenew(account, state, state.generation);
-    return "acquired";
-  }
+  const watchLease = createWatchLease({
+    lock: deps.lock,
+    ttlSeconds: leaseTtlSeconds,
+    renewIntervalMs,
+    watchers,
+    isRunning: () => running,
+    stopWatcher,
+    log,
+  });
 
   // kumiko-lint-ignore complexity-budget watch lifecycle (start/stop/restart) stays cohesive
   async function ensureWatcher(
-    account: MailAccountRecord,
+    account: SyncableAccount,
     plugin: InboundMailProviderPlugin,
   ): Promise<void> {
     // skip: provider without live push; the reconciliation poll covers the account.
     if (!plugin.watch) return;
+    const stale = watchers.get(account.id);
+    if (stale?.stop && stale.trustedAuthservId !== (account.trustedAuthservId ?? null)) {
+      await stopWatcher(account.id);
+    }
     const existing = watchers.get(account.id);
     // skip: the watcher is already running, starting, or scheduled for restart.
     if (existing?.stop || existing?.restartTimer || existing?.starting) return;
@@ -613,29 +319,23 @@ export function createInboundMailSupervisor(
       renewTimer: null,
       renewFailures: 0,
       starting: false,
+      trustedAuthservId: null,
     };
     watchers.set(account.id, state);
+    state.starting = true;
 
-    const leaseResult = await acquireWatchLease(account, state);
-    if (leaseResult === "no-token") {
-      // Another worker already holds this account's watch lease — the poll
-      // still covers it, this worker just doesn't open a second IDLE
-      // connection. Retried on the next tick.
-      if (!existing) watchers.delete(account.id);
-      // skip: no lease token acquired above — nothing more to set up here.
-      return;
-    }
-    if (leaseResult === "stale") {
-      // skip: state was retired by a concurrent stopWatcher while acquire()
-      // awaited — committing here would leak the lease.
+    const leaseResult = await watchLease.acquire(account, state);
+    if (leaseResult !== "acquired") {
+      if (leaseResult === "no-token" && !existing) watchers.delete(account.id);
+      state.starting = false;
+      // skip: no-token means the poll still covers the account and the next tick retries; stale means stopWatcher retired this state.
       return;
     }
 
     const generation = state.generation;
 
     const scheduleRestart = (err: unknown) => {
-      // skip: the supervisor stopped or the watcher generation changed; a restart
-      // would create a zombie watcher.
+      // skip: stopped or superseded generation; a restart would create a zombie watcher.
       if (!running || state.generation !== generation) return;
       state.stop = null;
       const delay = state.backoffMs;
@@ -650,16 +350,16 @@ export function createInboundMailSupervisor(
       }, delay);
     };
 
-    state.starting = true;
+    state.trustedAuthservId = account.trustedAuthservId ?? null;
     try {
       const accessToken = await resolveAccessToken(account, plugin);
       const stop = await plugin.watch(
         providerCtx,
         account,
         {
-          onMessages: async (msgs) => {
+          onMessages: async (msgs, rejected = []) => {
             try {
-              await ingestBatch(account, msgs, "watch");
+              await ingester.ingestBatch(account, msgs, rejected, "watch");
             } catch (err) {
               // An ingest failure must not kill the watcher: the next poll
               // tick fetches the messages again and dedup keeps it idempotent.
@@ -698,6 +398,7 @@ export function createInboundMailSupervisor(
           `inbound-mail: markAccount(watching) for account ${account.id} failed: ${describeError(err)}`,
         );
       }
+      watchCatchUp.schedule(account, plugin);
     } catch (err) {
       const keepRunning = await handleSyncError(account, err);
       if (keepRunning) scheduleRestart(err);
@@ -767,17 +468,26 @@ export function createInboundMailSupervisor(
       // skip: already started; start() is idempotent.
       if (running) return;
       running = true;
+      unregisterWake = registerMailAccountWakeListener((_tenantId, accountId) => {
+        syncAccountNow(accountId).catch((err) =>
+          log(`inbound-mail: wake sync for account ${accountId} failed: ${describeError(err)}`),
+        );
+      });
       await pollOnce();
       scheduleNextPoll();
     },
     pollOnce,
+    syncAccountNow,
     async stop() {
       running = false;
+      unregisterWake?.();
+      unregisterWake = null;
       if (pollTimer) {
         clearTimeout(pollTimer);
         pollTimer = null;
       }
       if (pollInFlight) await pollInFlight;
+      await Promise.allSettled([...inFlightSyncs.values(), watchCatchUp.settled()]);
       for (const accountId of [...watchers.keys()]) {
         await stopWatcher(accountId);
       }