@cosmicdrift/kumiko-samples 0.392.2 → 0.394.0

package/packages/bundled-features/src/inbound-mail-foundation/content-refetch.ts

@@ -0,0 +1,137 @@
+// Reloads the content of mails that were imported as snippet only (event version 1, before full text
+// storage existed). Runs after a successful poll with a small per-account budget, so a large backlog
+// is worked off over several ticks without slowing the regular sync. "unavailable" is terminal:
+// mails that are gone from the server stop being candidates.
+
+import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
+import { selectMany } from "@cosmicdrift/kumiko-framework/db";
+import { Temporal } from "@cosmicdrift/kumiko-types/temporal";
+import { inboundMessageAggregateId } from "./aggregate-id.ts";
+import { InboundMailFoundationHandlers } from "./constants.ts";
+import { seenMessageTable } from "./entities.ts";
+import { normalizeRawInboundMessage } from "./normalize-inbound-message.ts";
+import { inboundMessagesProjectionTable } from "./projection.ts";
+import type { DispatchWrite } from "./supervisor-ingest.ts";
+import type { InboundMailContext, InboundMailProviderPlugin, MailAccountRecord } from "./types.ts";
+
+export const CONTENT_REFETCH_BUDGET_PER_TICK = 50;
+const SEEN_PAGE_SIZE = 250;
+const SEEN_MAX_PAGES = 4;
+
+type SeenRow = { readonly providerMessageId: string; readonly seenAt: Temporal.Instant };
+type MessageRow = {
+  readonly id: string;
+  readonly bodyRef?: string | null;
+  readonly contentState?: string | null;
+};
+
+function needsContent(row: MessageRow): boolean {
+  const bodyMissing = row.bodyRef === undefined || row.bodyRef === null || row.bodyRef === "";
+  return bodyMissing && (row.contentState == null || row.contentState === "snippet");
+}
+
+async function findSnippetOnlyMessageIds(
+  db: DbConnection,
+  account: MailAccountRecord,
+  windowDays: number,
+): Promise<string[]> {
+  const cutoff = Temporal.Now.instant().subtract({ hours: windowDays * 24 });
+  const found: string[] = [];
+  let before: Temporal.Instant | undefined;
+  for (
+    let page = 0;
+    page < SEEN_MAX_PAGES && found.length < CONTENT_REFETCH_BUDGET_PER_TICK;
+    page++
+  ) {
+    const seen = await selectMany<SeenRow>(
+      db,
+      seenMessageTable,
+      {
+        tenantId: account.tenantId,
+        accountId: account.id,
+        seenAt: before ? { gte: cutoff, lt: before } : { gte: cutoff },
+      },
+      {
+        fields: ["providerMessageId", "seenAt"],
+        orderBy: { col: "seenAt", direction: "desc" },
+        limit: SEEN_PAGE_SIZE,
+      },
+    );
+    if (seen.length === 0) break;
+    const aggregateToProvider = new Map(
+      seen.map((row) => [
+        inboundMessageAggregateId(account.id, row.providerMessageId),
+        row.providerMessageId,
+      ]),
+    );
+    const rows = await selectMany<MessageRow>(
+      db,
+      inboundMessagesProjectionTable,
+      { tenantId: account.tenantId, id: { in: [...aggregateToProvider.keys()] } },
+      { fields: ["id", "bodyRef", "contentState"] },
+    );
+    for (const row of rows) {
+      const providerMessageId = aggregateToProvider.get(row.id);
+      if (providerMessageId !== undefined && needsContent(row)) found.push(providerMessageId);
+    }
+    const oldest = seen[seen.length - 1];
+    if (seen.length < SEEN_PAGE_SIZE || !oldest) break;
+    before = oldest.seenAt;
+  }
+  return found.slice(0, CONTENT_REFETCH_BUDGET_PER_TICK);
+}
+
+export async function refetchSnippetOnlyMessages(args: {
+  readonly db: DbConnection;
+  readonly dispatchWrite: DispatchWrite;
+  readonly providerCtx: InboundMailContext;
+  readonly account: MailAccountRecord;
+  readonly plugin: InboundMailProviderPlugin;
+  readonly windowDays: number;
+  readonly accessToken?: string | undefined;
+  readonly signal?: AbortSignal | undefined;
+}): Promise<void> {
+  const { account, plugin, dispatchWrite } = args;
+  // skip: the provider cannot reload content.
+  if (!plugin.fetchContent) return;
+  const ids = await findSnippetOnlyMessageIds(args.db, account, args.windowDays);
+  // skip: no snippet-only mail to reload.
+  if (ids.length === 0) return;
+  const result = await plugin.fetchContent(
+    args.providerCtx,
+    account,
+    ids,
+    args.accessToken !== undefined || args.signal !== undefined
+      ? {
+          ...(args.accessToken !== undefined && { accessToken: args.accessToken }),
+          ...(args.signal !== undefined && { signal: args.signal }),
+        }
+      : undefined,
+  );
+  const attach = (providerMessageId: string, content: Record<string, unknown>) =>
+    dispatchWrite({
+      handlerQn: InboundMailFoundationHandlers.attachMessageContent,
+      tenantId: account.tenantId,
+      payload: { accountId: account.id, providerMessageId, ...content },
+    });
+  for (const raw of result.messages) {
+    const msg = normalizeRawInboundMessage(raw);
+    if (msg.bodyText === "" && msg.rawMime == null && msg.bodyHtml === null) {
+      // Nothing usable came back: attach would leave the mail at "snippet", i.e. a candidate on
+      // every tick. "unavailable" is terminal.
+      await attach(msg.providerMessageId, { outcome: "unavailable" });
+      continue;
+    }
+    await attach(msg.providerMessageId, {
+      outcome: "fetched",
+      bodyText: msg.bodyText,
+      bodyTextTruncated: msg.bodyTextTruncated,
+      ...(msg.rawMime != null && { rawMime: msg.rawMime }),
+      ...(msg.bodyHtml !== null && { bodyHtml: msg.bodyHtml }),
+      attachments: msg.attachments,
+      skippedAttachments: msg.skippedAttachments,
+    });
+  }
+  const gone = [...result.missing, ...result.rejected.map((r) => r.providerMessageId)];
+  for (const providerMessageId of gone) await attach(providerMessageId, { outcome: "unavailable" });
+}

package/packages/bundled-features/src/inbound-mail-foundation/describe-error.ts

@@ -0,0 +1,5 @@
+// Provider errors can embed token or response data in `message`; logs carry
+// only the error class.
+export function describeError(err: unknown): string {
+  return err instanceof Error ? err.name : typeof err;
+}

package/packages/bundled-features/src/inbound-mail-foundation/entities.ts

@@ -2,7 +2,10 @@ import { collectPiiSubjectFields } from "@cosmicdrift/kumiko-framework/crypto";
 import { deriveEntityTableMeta } from "@cosmicdrift/kumiko-framework/db";
 import {
   access,
+  createBooleanField,
   createEntity,
+  createFilesField,
+  createLongTextField,
   createNumberField,
   createTextField,
   createTimestampField,
@@ -64,7 +67,15 @@ export const mailAccountEntity = createEntity({
       personal: false,
       reason: "technical_reference",
     }),
+    // Code of InboundMailLastErrors, never provider error text. Null on rows from before the column.
+    lastError: createTextField({ maxLength: 50, personal: false, reason: "technical_reference" }),
     watchState: createTextField({ maxLength: 100, personal: false, reason: "technical_reference" }),
+    // Lower-case authserv-id of the mailbox's own server; null = none set.
+    trustedAuthservId: createTextField({
+      maxLength: 255,
+      personal: false,
+      reason: "technical_reference",
+    }),
     connectedAt: createTimestampField({ required: true }),
   },
 });
@@ -82,8 +93,7 @@ export const inboundMessageEntity = createEntity({
       reason: "technical_reference",
     }),
     // Scope is inherited from the account at ingest time: messages from a
-    // personal mailbox are visible only to the owner (+ TenantAdmin),
-    // without joining read_mail_accounts.
+    // personal mailbox are visible only to the owner, without joining read_mail_accounts.
     ownerUserId: createTextField({ maxLength: 36, personal: false, reason: "pseudonymous_fk" }),
     messageIdHeader: createTextField({
       required: true,
@@ -106,7 +116,40 @@ export const inboundMessageEntity = createEntity({
     receivedAt: createTimestampField({ required: true }),
     bodyRef: createTextField({ maxLength: 500, personal: false, reason: "technical_reference" }),
     scope: createTextField({ maxLength: 200, personal: false, reason: "technical_reference" }),
+    // Full plaintext, capped at 32_000 characters before encryption (ciphertext is ~2.3x plus envelope).
+    // Null on rows from before the column; list-messages never selects it.
+    bodyText: createLongTextField({ maxLength: 200_000, personal: "tenant", find: "none" }),
+    bodyTextTruncated: createBooleanField(),
+    // Bound to the message (entityType = the aggregate type), so the file guard answers with the
+    // mail's visibility. The fileRefs are bound at ingest, the projection copies the ids.
+    attachmentFileIds: createFilesField({ personal: false, reason: "technical_reference" }),
+    // JSON-stringified SkippedInboundAttachment[]: nullable JSON stays a text column
+    // (jsonb fields are NOT NULL), like to/cc.
+    skippedAttachments: createTextField({
+      maxLength: 20_000,
+      personal: false,
+      reason: "technical_reference",
+    }),
+    // JSON-stringified SenderAuthentication or "null"; a SQL null (rows from before the column) reads as null.
+    senderAuthentication: createTextField({
+      maxLength: 4000,
+      personal: false,
+      reason: "technical_reference",
+    }),
+    // "complete" | "snippet" | "unavailable"; null counts as "snippet".
+    contentState: createTextField({
+      maxLength: 20,
+      personal: false,
+      reason: "technical_reference",
+    }),
+    // One of INBOUND_CONTENT_UNAVAILABLE_REASONS; null while the content is there or not yet missing.
+    contentUnavailableReason: createTextField({
+      maxLength: 30,
+      personal: false,
+      reason: "technical_reference",
+    }),
   },
+  attachedFileAccess: "record-only",
 });
 
 /** Thread-Rollup. PK = aggregateId = mailThreadAggregateId(tenantId,

package/packages/bundled-features/src/inbound-mail-foundation/events.ts

@@ -22,6 +22,7 @@
 
 import * as z from "zod";
 import { INBOUND_MAIL_FOUNDATION_FEATURE, InboundMailAccountStatuses } from "./constants.ts";
+import { senderAuthenticationFieldSchema } from "./sender-auth.ts";
 
 // Aggregate types for the event store.
 export const MAIL_ACCOUNT_AGGREGATE_TYPE = "mail-account" as const;
@@ -35,6 +36,8 @@ export const MAIL_ACCOUNT_UPDATED_EVENT_SHORT = "mail-account-updated" as const;
 export const MAIL_ACCOUNT_DISCONNECTED_EVENT_SHORT = "mail-account-disconnected" as const;
 export const INBOUND_MESSAGE_RECEIVED_EVENT_SHORT = "inbound-message-received" as const;
 export const MAIL_THREAD_UPDATED_EVENT_SHORT = "mail-thread-updated" as const;
+export const INBOUND_MESSAGE_CONTENT_FETCHED_EVENT_SHORT =
+  "inbound-message-content-fetched" as const;
 
 export const MAIL_ACCOUNT_CONNECTED_EVENT_QN =
   `${INBOUND_MAIL_FOUNDATION_FEATURE}:event:${MAIL_ACCOUNT_CONNECTED_EVENT_SHORT}` as const;
@@ -44,6 +47,8 @@ export const MAIL_ACCOUNT_DISCONNECTED_EVENT_QN =
   `${INBOUND_MAIL_FOUNDATION_FEATURE}:event:${MAIL_ACCOUNT_DISCONNECTED_EVENT_SHORT}` as const;
 export const INBOUND_MESSAGE_RECEIVED_EVENT_QN =
   `${INBOUND_MAIL_FOUNDATION_FEATURE}:event:${INBOUND_MESSAGE_RECEIVED_EVENT_SHORT}` as const;
+export const INBOUND_MESSAGE_CONTENT_FETCHED_EVENT_QN =
+  `${INBOUND_MAIL_FOUNDATION_FEATURE}:event:${INBOUND_MESSAGE_CONTENT_FETCHED_EVENT_SHORT}` as const;
 export const MAIL_THREAD_UPDATED_EVENT_QN =
   `${INBOUND_MAIL_FOUNDATION_FEATURE}:event:${MAIL_THREAD_UPDATED_EVENT_SHORT}` as const;
 
@@ -76,9 +81,51 @@ export const mailAccountEventPayloadSchema = z.object({
   /** Provider-agnostic watch state ("idle", "watching", "backoff:3", ...)
    *  — an operational signal only, not PII. */
   watchState: z.string().max(100),
+  /** Why the mailbox is degraded (InboundMailLastErrors code); null while healthy. A string rather
+   *  than an enum, like authMethod, so a new code needs no upcaster. */
+  lastError: z.string().max(50).nullable(),
+  /** authserv-id (lower case) of the mailbox's own server; only an Authentication-Results header
+   *  with exactly this id is believed (sender-auth.ts). null = not set, no sender verdicts. */
+  trustedAuthservId: z.string().max(255).nullable(),
 });
 export type MailAccountEventPayload = z.infer<typeof mailAccountEventPayloadSchema>;
 
+export const INBOUND_CONTENT_STATES = ["complete", "snippet", "unavailable"] as const;
+export type InboundContentState = (typeof INBOUND_CONTENT_STATES)[number];
+
+/** Why a message has no content: held back by the quarantine (retry possible), no file storage wired, or gone from the server. */
+export const INBOUND_CONTENT_UNAVAILABLE_REASONS = [
+  "quarantined",
+  "no_file_storage",
+  "gone_from_server",
+] as const;
+export type InboundContentUnavailableReason = (typeof INBOUND_CONTENT_UNAVAILABLE_REASONS)[number];
+
+export const skippedAttachmentSchema = z.object({
+  mimeType: z.string().max(255),
+  size: z.number().int().min(0),
+  reason: z.enum([
+    "too_large",
+    "mail_too_large",
+    "rejected",
+    "no_file_storage",
+    "nesting_too_deep",
+  ]),
+});
+
+// Shared by inbound-message-received (v2+) and inbound-message-content-fetched.
+// bodyText is ciphertext: 32_000 characters become roughly 75 KB, hence the generous max.
+const inboundMessageContentShape = {
+  bodyText: z.string().max(200_000),
+  bodyTextTruncated: z.boolean(),
+  /** file-foundation FileRef ids (plaintext uuids). Attachments outlive the mail: apps turn them
+   *  into documents, so the retention sweep never deletes them. */
+  attachmentFileIds: z.array(z.uuid()).max(50),
+  skippedAttachments: z.array(skippedAttachmentSchema).max(200),
+  contentState: z.enum(INBOUND_CONTENT_STATES),
+  contentUnavailableReason: z.enum(INBOUND_CONTENT_UNAVAILABLE_REASONS).nullable(),
+};
+
 // ============================================================================
 // inbound-message stream — exactly ONE received event per message
 // ============================================================================
@@ -110,15 +157,35 @@ export const inboundMessageEventPayloadSchema = z.object({
   /** ISO instant — receipt time reported by the provider (INTERNALDATE or
    *  receivedDateTime), not ingest time. */
   receivedAtIso: z.string().min(1),
-  /** file-foundation reference to the raw body (MIME). Empty when the app
-   *  is mounted without body persistence (snippet-only mode). */
+  /** file-foundation reference to the original (.eml, or .html for HTML-only mails). Empty for
+   *  snippet-only mails and when no file storage is available. */
   bodyRef: z.string().max(500),
   /** Domain scope hint for downstream business processes — generic
    *  ("inbox", folder name), with NO app-specific semantics. */
   scope: z.string().max(200),
+  /** SPF/DKIM/DMARC as reported by the mailbox's own server; null when there is no trustworthy
+   *  statement. Not PII. */
+  senderAuthentication: senderAuthenticationFieldSchema,
+  ...inboundMessageContentShape,
 });
 export type InboundMessageEventPayload = z.infer<typeof inboundMessageEventPayloadSchema>;
 
+// ============================================================================
+// Content of a message that was imported as snippet only and re-downloaded later.
+// Same aggregate as the received event; applies as an update of the projection row.
+// ============================================================================
+export const inboundMessageContentFetchedEventPayloadSchema = z.object({
+  /** Account and scope of the message, so a subscriber can route the event without loading the
+   *  stream. Null on events stored before v3: the stream id cannot be reversed to the account. */
+  accountId: z.uuid().nullable(),
+  ownerUserId: z.string().max(36).nullable(),
+  bodyRef: z.string().max(500),
+  ...inboundMessageContentShape,
+});
+export type InboundMessageContentFetchedEventPayload = z.infer<
+  typeof inboundMessageContentFetchedEventPayloadSchema
+>;
+
 // ============================================================================
 // mail-thread-Stream — Thread-Rollup pro (tenantId, threadKey)
 // ============================================================================