@cosmicdrift/kumiko-samples 0.392.2 → 0.394.0

package/packages/bundled-features/src/inbound-mail-foundation/account-health.ts

@@ -0,0 +1,146 @@
+// Account health: maps what the supervisor observes (repeated poll failures, quarantined mails, file
+// storage, KMS or ingest outage) onto the account status "degraded" plus a lastError code, and back to "active" once
+// nothing is wrong any more. Auth errors and disconnects are never touched here. An update is
+// dispatched only when status or lastError really changes, so a healthy mailbox causes no events.
+
+import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
+import {
+  type InboundMailAccountStatus,
+  InboundMailAccountStatuses,
+  InboundMailFoundationHandlers,
+  type InboundMailLastError,
+  InboundMailLastErrors,
+} from "./constants.ts";
+import { hasOpenQuarantine } from "./quarantine.ts";
+import type {
+  DispatchWrite,
+  IngestBatchOutcome,
+  IngestBlockedReason,
+} from "./supervisor-ingest.ts";
+import type { AccountHealthRef, SyncableAccount } from "./syncable-accounts.ts";
+
+const FAILURES_BEFORE_DEGRADED = 3;
+const OUTAGE_LAST_ERRORS: ReadonlySet<string> = new Set([
+  InboundMailLastErrors.encryptionUnavailable,
+  InboundMailLastErrors.ingestUnavailable,
+  InboundMailLastErrors.fileStorageUnavailable,
+]);
+const SUPERVISOR_AUDIT_REASON = "watch_supervisor";
+
+export type PollFailureKind = "transient" | "rate_limited";
+
+type AppliedHealth = {
+  readonly status: InboundMailAccountStatus;
+  readonly lastError: string | null;
+};
+
+export type AccountHealth = {
+  readonly onPollSuccess: (
+    account: SyncableAccount,
+    outcome: Pick<IngestBatchOutcome, "storageUnavailable" | "storageHealthy">,
+  ) => Promise<void>;
+  readonly onPollFailure: (account: SyncableAccount, kind: PollFailureKind) => Promise<void>;
+  /** The cursor stayed before a failing mail: either the mail waits for its count or the ingest is down. */
+  readonly onIngestBlocked: (
+    account: SyncableAccount,
+    blocked: IngestBlockedReason,
+    outcome: Pick<IngestBatchOutcome, "storageHealthy">,
+  ) => Promise<void>;
+  /** The address did not decrypt because the key store is down, so the account could not be polled. */
+  readonly onKeyStoreUnavailable: (account: AccountHealthRef) => Promise<void>;
+};
+
+export function createAccountHealth(deps: {
+  readonly db: DbConnection;
+  readonly dispatchWrite: DispatchWrite;
+  readonly log: (line: string) => void;
+}): AccountHealth {
+  const consecutiveFailures = new Map<string, number>();
+  const storageUnavailable = new Set<string>();
+  // The account snapshot of a tick is stale after our own update; this keeps the "real change" check honest.
+  // Per process: a drift between workers costs at most one redundant update-account.
+  const applied = new Map<string, AppliedHealth>();
+
+  async function apply(account: AccountHealthRef, lastError: InboundMailLastError | null) {
+    const known = applied.get(account.id) ?? {
+      status: account.status,
+      lastError: account.lastError,
+    };
+    const status =
+      lastError === null ? InboundMailAccountStatuses.active : InboundMailAccountStatuses.degraded;
+    // skip: the stored health already matches, nothing to write.
+    if (known.status === status && known.lastError === lastError) return;
+    const result = await deps.dispatchWrite({
+      handlerQn: InboundMailFoundationHandlers.updateAccount,
+      tenantId: account.tenantId,
+      payload: { accountId: account.id, status, lastError, reason: SUPERVISOR_AUDIT_REASON },
+    });
+    if (!result.isSuccess) {
+      deps.log(
+        `inbound-mail: account ${account.id} health update was rejected (${rejectionCode(result.error)})`,
+      );
+      // skip: not recorded as applied, so the next tick retries the update.
+      return;
+    }
+    applied.set(account.id, { status, lastError });
+  }
+
+  async function lastErrorWithoutOutage(account: AccountHealthRef) {
+    if (await hasOpenQuarantine(deps.db, account.id)) {
+      return InboundMailLastErrors.messageQuarantined;
+    }
+    if (storageUnavailable.has(account.id)) return InboundMailLastErrors.fileStorageUnavailable;
+    return null;
+  }
+
+  return {
+    async onPollSuccess(account, outcome) {
+      consecutiveFailures.delete(account.id);
+      if (outcome.storageUnavailable) storageUnavailable.add(account.id);
+      else if (outcome.storageHealthy) storageUnavailable.delete(account.id);
+      return apply(account, await lastErrorWithoutOutage(account));
+    },
+    async onIngestBlocked(account, blocked, outcome) {
+      consecutiveFailures.delete(account.id);
+      if (blocked === "message_failing") {
+        deps.log(`inbound-mail: account ${account.id} waits for a failing message to be counted`);
+        if (outcome.storageHealthy) storageUnavailable.delete(account.id);
+        // A counted failure proves the ingest works again, so a shown outage is over; anything else stays.
+        const shown = (applied.get(account.id) ?? account).lastError;
+        // skip: no outage is shown, the status stays as it is
+        if (shown === null || !OUTAGE_LAST_ERRORS.has(shown)) return;
+        return apply(account, await lastErrorWithoutOutage(account));
+      }
+      // KMS and ingest outages need no memory like storage: the next unblocked poll proves they are over.
+      if (blocked === "encryption_unavailable") {
+        return apply(account, InboundMailLastErrors.encryptionUnavailable);
+      }
+      if (blocked === "ingest_unavailable") {
+        return apply(account, InboundMailLastErrors.ingestUnavailable);
+      }
+      storageUnavailable.add(account.id);
+      return apply(account, InboundMailLastErrors.fileStorageUnavailable);
+    },
+    async onKeyStoreUnavailable(account) {
+      consecutiveFailures.delete(account.id);
+      return apply(account, InboundMailLastErrors.encryptionUnavailable);
+    },
+    async onPollFailure(account, kind) {
+      const count = (consecutiveFailures.get(account.id) ?? 0) + 1;
+      consecutiveFailures.set(account.id, count);
+      // skip: below the failure threshold the mailbox stays as it is.
+      if (count < FAILURES_BEFORE_DEGRADED) return;
+      return apply(
+        account,
+        kind === "rate_limited"
+          ? InboundMailLastErrors.rateLimited
+          : InboundMailLastErrors.connectionFailed,
+      );
+    },
+  };
+}
+
+function rejectionCode(error: unknown): string {
+  if (typeof error === "object" && error !== null && "code" in error) return String(error.code);
+  return "unknown";
+}

package/packages/bundled-features/src/inbound-mail-foundation/account-poll-schedule.ts

@@ -0,0 +1,74 @@
+// How the supervisor tick spreads its accounts: a bounded number of syncs side by side, and a
+// deadline after which a sync that never answers is abandoned instead of holding the tick.
+
+import type { PollAccountOptions, PollAccountResult } from "./sync-account-now.ts";
+import type { SyncableAccount } from "./syncable-accounts.ts";
+import type { InboundMailProviderPlugin } from "./types.ts";
+
+export const DEFAULT_POLL_CONCURRENCY = 4;
+export const DEFAULT_ACCOUNT_POLL_DEADLINE_MS = 10 * 60 * 1000;
+
+export function pollScheduleOf(options: {
+  readonly pollConcurrency?: number;
+  readonly accountPollDeadlineMs?: number;
+}): { readonly pollConcurrency: number; readonly accountPollDeadlineMs: number } {
+  return {
+    pollConcurrency: options.pollConcurrency ?? DEFAULT_POLL_CONCURRENCY,
+    accountPollDeadlineMs: options.accountPollDeadlineMs ?? DEFAULT_ACCOUNT_POLL_DEADLINE_MS,
+  };
+}
+
+type DeadlineOutcome<T> =
+  | { readonly timedOut: false; readonly value: T }
+  | { readonly timedOut: true };
+
+/** Aborts the signal when the deadline passes. The abandoned work may still settle later; its
+ *  outcome is dropped and a late rejection is swallowed. */
+async function runWithDeadline<T>(
+  deadlineMs: number,
+  run: (signal: AbortSignal) => Promise<T>,
+): Promise<DeadlineOutcome<T>> {
+  const controller = new AbortController();
+  const work = run(controller.signal);
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const deadline = new Promise<DeadlineOutcome<T>>((resolve) => {
+    timer = setTimeout(() => resolve({ timedOut: true }), deadlineMs);
+  });
+  try {
+    const outcome = await Promise.race([
+      work.then((value): DeadlineOutcome<T> => ({ timedOut: false, value })),
+      deadline,
+    ]);
+    if (outcome.timedOut) {
+      controller.abort();
+      work.catch(() => {});
+    }
+    return outcome;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+type PollAccount = (
+  account: SyncableAccount,
+  plugin: InboundMailProviderPlugin,
+  options: PollAccountOptions,
+) => Promise<PollAccountResult>;
+
+/** A sync past the deadline counts as one failed poll and its in-flight slot is freed. */
+export function withPollDeadline(
+  pollAccount: PollAccount,
+  deadline: {
+    readonly ms: number;
+    readonly onDeadline: (account: SyncableAccount) => Promise<void>;
+  },
+): PollAccount {
+  return async (account, plugin, options) => {
+    const outcome = await runWithDeadline(deadline.ms, (signal) =>
+      pollAccount(account, plugin, { ...options, signal }),
+    );
+    if (!outcome.timedOut) return outcome.value;
+    await deadline.onDeadline(account);
+    return "done";
+  };
+}

package/packages/bundled-features/src/inbound-mail-foundation/account-poll.ts

@@ -0,0 +1,238 @@
+import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
+import { selectMany } from "@cosmicdrift/kumiko-framework/db";
+import type { AccountHealth, PollFailureKind } from "./account-health.ts";
+import { refetchSnippetOnlyMessages } from "./content-refetch.ts";
+import { describeError } from "./describe-error.ts";
+import { seenMessageTable } from "./entities.ts";
+import {
+  isInboundAuthError,
+  isInboundCredentialNotStoredError,
+  isInboundCursorInvalidError,
+  isInboundRateLimitError,
+} from "./provider-errors.ts";
+import { quarantinedMessageIds } from "./quarantine.ts";
+import { retryQuarantinedMessages } from "./quarantine-retry.ts";
+import type {
+  DispatchWrite,
+  IngestBatchOutcome,
+  IngestBlockedReason,
+  MessageIngester,
+} from "./supervisor-ingest.ts";
+import type { PollAccountOptions, PollAccountResult } from "./sync-account-now.ts";
+import type { SyncCursorStore } from "./sync-cursor-store.ts";
+import type { SyncableAccount } from "./syncable-accounts.ts";
+import type { InboundMailContext, InboundMailProviderPlugin, MailAccountRecord } from "./types.ts";
+
+const SEEN_LOOKUP_CHUNK_SIZE = 1000;
+
+export type AccountPollerDeps = {
+  readonly systemDb: DbConnection;
+  readonly cursors: SyncCursorStore;
+  readonly ingester: MessageIngester;
+  readonly health: AccountHealth;
+  readonly providerCtx: InboundMailContext;
+  readonly dispatchWrite: DispatchWrite;
+  readonly backfillWindowDays: number;
+  readonly maxMessagesPerPoll: number;
+  readonly resolveAccessToken: (
+    account: MailAccountRecord,
+    plugin: InboundMailProviderPlugin,
+  ) => Promise<string | undefined>;
+  /** Returns true when the account may keep running. */
+  readonly handleSyncError: (account: MailAccountRecord, err: unknown) => Promise<boolean>;
+  readonly log: (line: string) => void;
+};
+
+export function createAccountPoller(deps: AccountPollerDeps): {
+  readonly pollAccount: (
+    account: SyncableAccount,
+    plugin: InboundMailProviderPlugin,
+    options?: PollAccountOptions,
+  ) => Promise<PollAccountResult>;
+} {
+  const {
+    systemDb,
+    cursors,
+    ingester,
+    health,
+    providerCtx,
+    backfillWindowDays,
+    maxMessagesPerPoll,
+    resolveAccessToken,
+    handleSyncError,
+    log,
+  } = deps;
+
+  function unknownMessageIdsFilterFor(account: MailAccountRecord) {
+    return async (providerMessageIds: readonly string[]): Promise<ReadonlySet<string>> => {
+      if (providerMessageIds.length === 0) return new Set();
+      // One bind parameter per id: a full-mailbox listing would exceed Postgres' 65535 limit.
+      const seenIds = new Set<string>();
+      for (let start = 0; start < providerMessageIds.length; start += SEEN_LOOKUP_CHUNK_SIZE) {
+        const seen = await selectMany<{ providerMessageId: string }>(
+          systemDb,
+          seenMessageTable,
+          {
+            tenantId: account.tenantId,
+            accountId: account.id,
+            providerMessageId: {
+              in: providerMessageIds.slice(start, start + SEEN_LOOKUP_CHUNK_SIZE),
+            },
+          },
+          { fields: ["providerMessageId"] },
+        );
+        for (const row of seen) seenIds.add(row.providerMessageId);
+      }
+      // A quarantined message counts as seen: re-fetching it would only fail the same way again.
+      for (const id of await quarantinedMessageIds(systemDb, account.id, providerMessageIds)) {
+        seenIds.add(id);
+      }
+      return new Set(providerMessageIds.filter((id) => !seenIds.has(id)));
+    };
+  }
+
+  function pollFailureKindOf(err: unknown): PollFailureKind | null {
+    if (isInboundAuthError(err) || isInboundCursorInvalidError(err)) return null;
+    return isInboundRateLimitError(err) ? "rate_limited" : "transient";
+  }
+
+  async function refetchContentSafely(
+    account: SyncableAccount,
+    plugin: InboundMailProviderPlugin,
+    signal: AbortSignal | undefined,
+  ): Promise<void> {
+    // skip: the provider cannot reload content.
+    if (!plugin.fetchContent) return;
+    try {
+      const accessToken = await resolveAccessToken(account, plugin);
+      await refetchSnippetOnlyMessages({
+        db: systemDb,
+        dispatchWrite: deps.dispatchWrite,
+        providerCtx,
+        account,
+        plugin,
+        windowDays: backfillWindowDays,
+        accessToken,
+        signal,
+      });
+    } catch (err) {
+      // Best effort: the sync itself succeeded, the next tick retries the reload.
+      log(`inbound-mail: content reload for account ${account.id} failed: ${describeError(err)}`);
+    }
+  }
+
+  async function retryQuarantineSafely(
+    account: SyncableAccount,
+    plugin: InboundMailProviderPlugin,
+    signal: AbortSignal | undefined,
+  ): Promise<void> {
+    try {
+      const accessToken = await resolveAccessToken(account, plugin);
+      await retryQuarantinedMessages({
+        db: systemDb,
+        dispatchWrite: deps.dispatchWrite,
+        providerCtx,
+        account,
+        plugin,
+        accessToken,
+        signal,
+        log,
+      });
+    } catch (err) {
+      // Best effort: the sync itself succeeded, the flag stays and the next tick retries.
+      log(`inbound-mail: quarantine retry for account ${account.id} failed: ${describeError(err)}`);
+    }
+  }
+
+  // Fetches and ingests batch after batch until the provider has no more or the message budget of
+  // this tick is used up. An abandoned sync (deadline) throws out of the loop.
+  async function ingestBatches(
+    account: SyncableAccount,
+    plugin: InboundMailProviderPlugin,
+    options: PollAccountOptions,
+  ): Promise<{
+    storageUnavailable: boolean;
+    storageHealthy: boolean;
+    ingested: boolean;
+    blocked?: IngestBlockedReason;
+  }> {
+    const { signal } = options;
+    let cursor = await cursors.load(account.id);
+    let budget = maxMessagesPerPoll;
+    const totals = { storageUnavailable: false, storageHealthy: false, ingested: false };
+    for (;;) {
+      const accessToken = await resolveAccessToken(account, plugin);
+      const result = await plugin.fetch(providerCtx, account, cursor, {
+        backfillWindowDays,
+        maxMessages: budget,
+        filterUnknownMessageIds: unknownMessageIdsFilterFor(account),
+        ...(accessToken !== undefined && { accessToken }),
+        ...(signal && { signal }),
+      });
+      // The sync was abandoned while the provider was busy; its late result is dropped.
+      signal?.throwIfAborted();
+      const outcome: IngestBatchOutcome = await ingester.ingestBatch(
+        account,
+        result.messages,
+        result.rejected ?? [],
+        JSON.stringify(result.nextCursor),
+        {
+          ...(options.countBucket !== undefined && { countBucket: options.countBucket }),
+          ...(signal && { signal }),
+          proof: { ingested: totals.ingested, storageHealthy: totals.storageHealthy },
+        },
+      );
+      totals.storageUnavailable ||= outcome.storageUnavailable;
+      totals.storageHealthy ||= outcome.storageHealthy;
+      totals.ingested ||= outcome.ingested;
+      // Blocked: the cursor stays before the stuck mail, the next tick fetches it again.
+      if (outcome.blocked) return { ...totals, blocked: outcome.blocked };
+      // A newer poll may already own the account; an abandoned one must not move its cursor back.
+      signal?.throwIfAborted();
+      await cursors.save(account, result.nextCursor);
+      cursor = result.nextCursor;
+      budget -= result.messages.length;
+      if (!result.hasMore || budget <= 0) return totals;
+    }
+  }
+
+  async function pollAccount(
+    account: SyncableAccount,
+    plugin: InboundMailProviderPlugin,
+    options: PollAccountOptions = {},
+  ): Promise<PollAccountResult> {
+    let totals: Awaited<ReturnType<typeof ingestBatches>>;
+    try {
+      totals = await ingestBatches(account, plugin, options);
+    } catch (err) {
+      // skip: an error caused by abandoning the sync is already counted as the deadline failure.
+      if (options.signal?.aborted) return "done";
+      if (options.fromWake && isInboundCredentialNotStoredError(err)) {
+        // The secret is written after the account is created; the next tick picks it up.
+        log(`inbound-mail: account ${account.id} credential not stored yet, wake sync skipped`);
+        return "credential_pending";
+      }
+      const stillRunning = await handleSyncError(account, err);
+      const failureKind = pollFailureKindOf(err);
+      if (stillRunning && failureKind) await health.onPollFailure(account, failureKind);
+      return "done";
+    }
+    // skip: abandoned during the last batch; the deadline already counted this poll as failed.
+    if (options.signal?.aborted) return "done";
+    if (totals.blocked) {
+      await health.onIngestBlocked(account, totals.blocked, totals);
+      // A retry asked for by an admin does not depend on the cursor, so a stuck mail must not hold it.
+      // During an outage it would fail the same way; the flag stays and runs after the recovery.
+      if (totals.blocked === "message_failing") {
+        await retryQuarantineSafely(account, plugin, options.signal);
+      }
+      return "done";
+    }
+    await health.onPollSuccess(account, totals);
+    await retryQuarantineSafely(account, plugin, options.signal);
+    await refetchContentSafely(account, plugin, options.signal);
+    return "done";
+  }
+
+  return { pollAccount };
+}

package/packages/bundled-features/src/inbound-mail-foundation/account-record.ts

@@ -16,6 +16,7 @@ export type MailAccountRow = {
   readonly displayName?: string | null;
   readonly status?: string | null;
   readonly watchState?: string | null;
+  readonly trustedAuthservId?: string | null;
 };
 
 function isAccountStatus(value: unknown): value is InboundMailAccountStatus {
@@ -44,5 +45,6 @@ export async function toMailAccountRecord(
     displayName: row.displayName ?? "",
     status: isAccountStatus(row.status) ? row.status : InboundMailAccountStatuses.active,
     watchState: row.watchState ?? "idle",
+    trustedAuthservId: row.trustedAuthservId ?? null,
   };
 }

package/packages/bundled-features/src/inbound-mail-foundation/account-wake.ts

@@ -0,0 +1,21 @@
+// Process-local wake channel between the account-event consumer (feature.ts) and the sync supervisor.
+// A freshly connected, reconnected or re-activated mailbox is synced right away instead of waiting
+// for the next poll tick. The consumer runs "per-instance", so every process wakes its own supervisor.
+//
+// Without a running event dispatcher in the process (or without a supervisor) nothing wakes anyone;
+// the regular poll tick then picks the account up, so this is purely a latency optimisation.
+
+export type MailAccountWakeListener = (tenantId: string, accountId: string) => void;
+
+const listeners = new Set<MailAccountWakeListener>();
+
+export function registerMailAccountWakeListener(listener: MailAccountWakeListener): () => void {
+  listeners.add(listener);
+  return () => {
+    listeners.delete(listener);
+  };
+}
+
+export function wakeMailAccount(tenantId: string, accountId: string): void {
+  for (const listener of listeners) listener(tenantId, accountId);
+}