@open-workspace/email 0.0.0-stage → 0.1.1

package/worker/mailbox/schema.ts

@@ -0,0 +1,353 @@
+import {
+  index,
+  integer,
+  primaryKey,
+  sqliteTable,
+  text,
+  uniqueIndex,
+} from "drizzle-orm/sqlite-core";
+import { transportStates } from "../../shared/mail";
+import type { MailRuleActions, MailRuleConditions } from "../../shared/mail-rules";
+import { mailboxStates } from "./folder-model";
+import type {
+  DeliveryFacts,
+  EmailAiClassification,
+  EmailScreening,
+  ListUnsubscribe,
+  MailAddress,
+  RecipientDeliveryStatus,
+  SenderIntel,
+  StoredAttachment,
+  StoredEmailAuthentication,
+} from "./model";
+
+/**
+ * Only custom classification folders are stored. Inbox, Sent, Archive, Spam,
+ * and Trash are computed views over `conversations`.
+ */
+export const folders = sqliteTable(
+  "folders",
+  {
+    id: text("id").primaryKey(),
+    name: text("name").notNull(),
+    sortOrder: integer("sort_order").notNull().default(100),
+  },
+  (table) => [
+    uniqueIndex("folders_name_unique").on(table.name),
+    index("folders_order_idx").on(table.sortOrder, table.name),
+  ],
+);
+
+/**
+ * A row exists only while raw MIME still needs to be materialized as an email.
+ * Deterministic MIME errors are converted into a visible fallback message;
+ * this queue is reserved for retryable infrastructure failures.
+ */
+export const pendingInbound = sqliteTable(
+  "pending_inbound",
+  {
+    id: text("id").primaryKey(),
+    rawObjectKey: text("raw_object_key").notNull(),
+    envelopeFrom: text("envelope_from").notNull(),
+    envelopeTo: text("envelope_to").notNull(),
+    /** The workspace mailing list this copy came through, if any. */
+    listAddress: text("list_address"),
+    receivedAt: integer("received_at", { mode: "timestamp_ms" }).notNull(),
+    attempts: integer("attempts").notNull().default(0),
+    aiAttempts: integer("ai_attempts").notNull().default(0),
+    nextAttemptAt: integer("next_attempt_at", { mode: "timestamp_ms" }).notNull(),
+  },
+  (table) => [
+    index("pending_inbound_next_attempt_idx").on(
+      table.nextAttemptAt,
+      table.receivedAt,
+    ),
+  ],
+);
+
+/**
+ * R2 cleanup is retried independently after a permanent conversation delete.
+ * The row is committed in the same SQLite transaction as the metadata delete,
+ * so a Worker interruption cannot leave untracked message objects behind.
+ */
+export const pendingObjectDeletions = sqliteTable(
+  "pending_object_deletions",
+  {
+    objectKey: text("object_key").primaryKey(),
+    attempts: integer("attempts").notNull().default(0),
+    nextAttemptAt: integer("next_attempt_at", { mode: "timestamp_ms" }).notNull(),
+  },
+  (table) => [
+    index("pending_object_deletions_next_attempt_idx").on(table.nextAttemptAt),
+  ],
+);
+
+export const pendingOfficePreviews = sqliteTable(
+  "pending_office_previews",
+  {
+    sourceKey: text("source_key").primaryKey(),
+    emailId: text("email_id").notNull(),
+    attachmentId: text("attachment_id").notNull(),
+    attempts: integer("attempts").notNull().default(0),
+    nextAttemptAt: integer("next_attempt_at", { mode: "timestamp_ms" }).notNull(),
+  },
+  (table) => [index("pending_office_previews_next_attempt_idx").on(table.nextAttemptAt)],
+);
+
+/**
+ * One row is one message visible in this Durable Object's mailbox. RFC
+ * threading metadata and message content are immutable after insertion.
+ */
+export const emails = sqliteTable(
+  "emails",
+  {
+    id: text("id").primaryKey(),
+    requestFingerprint: text("request_fingerprint"),
+    conversationId: text("conversation_id").notNull(),
+    direction: text("direction", { enum: ["incoming", "outgoing"] }).notNull(),
+
+    messageIdHeader: text("message_id_header"),
+    inReplyToJson: text("in_reply_to_json", { mode: "json" })
+      .$type<string[]>()
+      .notNull()
+      .default([]),
+    referencesJson: text("references_json", { mode: "json" })
+      .$type<string[]>()
+      .notNull()
+      .default([]),
+
+    fromJson: text("from_json", { mode: "json" })
+      .$type<MailAddress[]>()
+      .notNull(),
+    replyToJson: text("reply_to_json", { mode: "json" })
+      .$type<MailAddress[]>()
+      .notNull()
+      .default([]),
+    toJson: text("to_json", { mode: "json" })
+      .$type<MailAddress[]>()
+      .notNull()
+      .default([]),
+    ccJson: text("cc_json", { mode: "json" })
+      .$type<MailAddress[]>()
+      .notNull()
+      .default([]),
+    bccJson: text("bcc_json", { mode: "json" })
+      .$type<MailAddress[]>()
+      .notNull()
+      .default([]),
+
+    subject: text("subject").notNull().default("(no subject)"),
+    preview: text("preview").notNull().default(""),
+    bodyText: text("body_text"),
+    quotedText: text("quoted_text"),
+    bodyHtmlR2Key: text("body_html_r2_key"),
+    rawMimeR2Key: text("raw_mime_r2_key"),
+    /** Header-derived sender authentication for incoming mail. */
+    authenticationResultsJson: text("authentication_results_json", { mode: "json" })
+      .$type<StoredEmailAuthentication>(),
+    /** SMTP envelope as delivered by Email Routing (MAIL FROM / RCPT TO). */
+    envelopeFrom: text("envelope_from"),
+    envelopeTo: text("envelope_to"),
+    listUnsubscribeJson: text("list_unsubscribe_json", { mode: "json" })
+      .$type<ListUnsubscribe>(),
+    /** DNS facts about the delivering server, gathered at ingestion. */
+    senderIntelJson: text("sender_intel_json", { mode: "json" })
+      .$type<SenderIntel>(),
+    aiClassificationJson: text("ai_classification_json", { mode: "json" })
+      .$type<EmailAiClassification>(),
+    /** Who the message was written for and whether it was forwarded, from ingestion. */
+    deliveryJson: text("delivery_json", { mode: "json" })
+      .$type<DeliveryFacts>(),
+    /** Block and rule decisions taken at ingestion, before AI classification. */
+    screeningJson: text("screening_json", { mode: "json" })
+      .$type<EmailScreening>(),
+    /** Set when someone in this mailbox reported the message. */
+    reportLabel: text("report_label", { enum: ["spam", "phishing"] }),
+    attachmentsJson: text("attachments_json", { mode: "json" })
+      .$type<StoredAttachment[]>()
+      .notNull()
+      .default([]),
+
+    listId: text("list_id"),
+    listPostAddress: text("list_post_address"),
+    timelineAt: integer("timeline_at", { mode: "timestamp_ms" }).notNull(),
+
+    transportState: text("transport_state", { enum: transportStates }).notNull(),
+    transportError: text("transport_error"),
+    deliveryStatusJson: text("delivery_status_json", { mode: "json" })
+      .$type<RecipientDeliveryStatus[]>()
+      .notNull()
+      .default([]),
+  },
+  (table) => [
+    index("emails_conversation_timeline_idx").on(
+      table.conversationId,
+      table.timelineAt,
+      table.id,
+    ),
+    index("emails_message_id_header_idx").on(table.messageIdHeader),
+  ],
+);
+
+/**
+ * Small mutable projection used by folder lists. A single insert trigger keeps
+ * it synchronized with immutable email rows.
+ */
+export const conversations = sqliteTable(
+  "conversations",
+  {
+    id: text("id").primaryKey(),
+    latestEmailId: text("latest_email_id")
+      .notNull()
+      .references(() => emails.id),
+    timelineAt: integer("timeline_at", { mode: "timestamp_ms" }).notNull(),
+    mailboxState: text("mailbox_state", { enum: mailboxStates })
+      .notNull()
+      .default("active"),
+    folderId: text("folder_id").references(() => folders.id, {
+      onDelete: "set null",
+    }),
+    hasIncoming: integer("has_incoming", { mode: "boolean" })
+      .notNull()
+      .default(false),
+    hasOutgoing: integer("has_outgoing", { mode: "boolean" })
+      .notNull()
+      .default(false),
+  },
+  (table) => [
+    index("conversations_timeline_idx").on(table.timelineAt, table.latestEmailId),
+    index("conversations_inbox_timeline_idx").on(
+      table.mailboxState,
+      table.hasIncoming,
+      table.timelineAt,
+      table.latestEmailId,
+    ),
+    index("conversations_sent_timeline_idx").on(
+      table.mailboxState,
+      table.hasOutgoing,
+      table.timelineAt,
+      table.latestEmailId,
+    ),
+    index("conversations_folder_timeline_idx").on(
+      table.folderId,
+      table.timelineAt,
+      table.latestEmailId,
+    ),
+  ],
+);
+
+/**
+ * Correspondent history of this mailbox, keyed by lowercased address. Inbound
+ * counters drive the first-time-sender hint; outbound counters record that the
+ * mailbox wrote to the address. `blocked_at` sends future mail to Spam;
+ * rows keyed `@domain` block a whole domain. `trusted_at` ("Not spam")
+ * suppresses caution-level hints and the AI spam verdict.
+ */
+export const senders = sqliteTable(
+  "senders",
+  {
+    address: text("address").primaryKey(),
+    domain: text("domain").notNull(),
+    firstSeenAt: integer("first_seen_at", { mode: "timestamp_ms" }),
+    lastSeenAt: integer("last_seen_at", { mode: "timestamp_ms" }),
+    messageCount: integer("message_count").notNull().default(0),
+    outboundCount: integer("outbound_count").notNull().default(0),
+    lastOutboundAt: integer("last_outbound_at", { mode: "timestamp_ms" }),
+    blockedAt: integer("blocked_at", { mode: "timestamp_ms" }),
+    trustedAt: integer("trusted_at", { mode: "timestamp_ms" }),
+    reportedAt: integer("reported_at", { mode: "timestamp_ms" }),
+  },
+  (table) => [index("senders_domain_idx").on(table.domain)],
+);
+
+/**
+ * Spam, phishing and not-spam reports made in this mailbox. They feed the
+ * AI classifier as sender history and are never sent anywhere by themselves.
+ */
+export const senderFeedback = sqliteTable(
+  "sender_feedback",
+  {
+    id: text("id").primaryKey(),
+    address: text("address").notNull(),
+    domain: text("domain").notNull(),
+    emailId: text("email_id"),
+    label: text("label", { enum: ["spam", "phishing", "not_spam"] }).notNull(),
+    userId: text("user_id"),
+    createdAt: integer("created_at", { mode: "timestamp_ms" }).notNull(),
+  },
+  (table) => [
+    index("sender_feedback_address_idx").on(table.address, table.createdAt),
+    index("sender_feedback_domain_idx").on(table.domain),
+  ],
+);
+
+/** Latest unsubscribe attempt per list (List-Id, else the From organizational domain). */
+export const unsubscriptions = sqliteTable("unsubscriptions", {
+  listKey: text("list_key").primaryKey(),
+  emailId: text("email_id"),
+  method: text("method", { enum: ["one_click", "mailto", "link"] }).notNull(),
+  status: text("status", { enum: ["done", "requested", "opened", "failed"] }).notNull(),
+  target: text("target"),
+  detail: text("detail"),
+  userId: text("user_id"),
+  createdAt: integer("created_at", { mode: "timestamp_ms" }).notNull(),
+});
+
+/** Ordered deterministic filters; they run at ingestion before AI sorting. */
+export const mailRules = sqliteTable(
+  "mail_rules",
+  {
+    id: text("id").primaryKey(),
+    name: text("name").notNull(),
+    sortOrder: integer("sort_order").notNull().default(100),
+    enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
+    conditionsJson: text("conditions_json", { mode: "json" }).$type<MailRuleConditions>().notNull(),
+    actionsJson: text("actions_json", { mode: "json" }).$type<MailRuleActions>().notNull(),
+    createdAt: integer("created_at", { mode: "timestamp_ms" }).notNull(),
+    updatedAt: integer("updated_at", { mode: "timestamp_ms" }).notNull(),
+  },
+  (table) => [index("mail_rules_order_idx").on(table.sortOrder, table.createdAt)],
+);
+
+export const mailboxAiConfiguration = sqliteTable(
+  "mailbox_ai_configuration",
+  {
+    id: text("id").primaryKey(),
+    instructions: text("instructions").notNull().default(""),
+    confidenceThreshold: integer("confidence_threshold").notNull().default(75),
+    updatedAt: integer("updated_at", { mode: "timestamp_ms" }).notNull(),
+  },
+);
+
+/**
+ * Read state belongs to a person, not to the mailbox. This keeps shared
+ * mailboxes independent: one teammate viewing a message cannot clear another
+ * teammate's unread state.
+ */
+export const emailReadStates = sqliteTable(
+  "email_read_states",
+  {
+    userId: text("user_id").notNull(),
+    emailId: text("email_id")
+      .notNull()
+      .references(() => emails.id, { onDelete: "cascade" }),
+    readAt: integer("read_at", { mode: "timestamp_ms" }).notNull(),
+  },
+  (table) => [
+    primaryKey({ columns: [table.userId, table.emailId] }),
+    index("email_read_states_email_idx").on(table.emailId),
+  ],
+);
+
+export type Email = typeof emails.$inferSelect;
+export type SenderRecord = typeof senders.$inferSelect;
+export type MailRuleRecord = typeof mailRules.$inferSelect;
+export type UnsubscriptionRecord = typeof unsubscriptions.$inferSelect;
+export type NewEmail = typeof emails.$inferInsert;
+export type EmailReadState = typeof emailReadStates.$inferSelect;
+export type PendingInbound = typeof pendingInbound.$inferSelect;
+export type PendingObjectDeletion = typeof pendingObjectDeletions.$inferSelect;
+export type NewPendingInbound = typeof pendingInbound.$inferInsert;
+export type ConversationRecord = typeof conversations.$inferSelect;
+export type FolderRecord = typeof folders.$inferSelect;
+export type NewFolder = typeof folders.$inferInsert;

package/worker/mailbox/senders.ts

@@ -0,0 +1,240 @@
+import {
+  blockPatternsFor,
+  type FeedbackLabel,
+  type SenderBlock,
+  type SenderStatus,
+} from "../../shared/mail-feedback";
+import type { SenderHistory } from "../mail/message-security";
+import type { NewEmail } from "./schema";
+
+function normalized(address: string | null | undefined) {
+  const value = address?.trim().toLocaleLowerCase() ?? "";
+  const at = value.lastIndexOf("@");
+  return at > 0 && at < value.length - 1 ? { address: value, domain: value.slice(at + 1) } : null;
+}
+
+function timestamp(value: NewEmail["timelineAt"]) {
+  return value instanceof Date ? value.getTime() : Number(value);
+}
+
+/**
+ * Upserts the correspondent counters for one stored message. Call it inside
+ * the transaction that inserted the row so counts never drift from `emails`.
+ */
+export function recordCorrespondents(sql: SqlStorage, email: NewEmail) {
+  const at = timestamp(email.timelineAt);
+  if (email.direction === "incoming") {
+    const sender = normalized(email.fromJson[0]?.address);
+    if (!sender) return;
+    sql.exec(
+      `insert into senders (address, domain, first_seen_at, last_seen_at, message_count)
+       values (?, ?, ?, ?, 1)
+       on conflict (address) do update set
+         first_seen_at = min(coalesce(first_seen_at, excluded.first_seen_at), excluded.first_seen_at),
+         last_seen_at = max(coalesce(last_seen_at, excluded.last_seen_at), excluded.last_seen_at),
+         message_count = message_count + 1`,
+      sender.address,
+      sender.domain,
+      at,
+      at,
+    );
+    return;
+  }
+  const recipients = new Map<string, string>();
+  for (const recipient of [...(email.toJson ?? []), ...(email.ccJson ?? []), ...(email.bccJson ?? [])]) {
+    const value = normalized(recipient.address);
+    if (value) recipients.set(value.address, value.domain);
+  }
+  for (const [address, domain] of recipients) {
+    sql.exec(
+      `insert into senders (address, domain, outbound_count, last_outbound_at)
+       values (?, ?, 1, ?)
+       on conflict (address) do update set
+         outbound_count = outbound_count + 1,
+         last_outbound_at = max(coalesce(last_outbound_at, excluded.last_outbound_at), excluded.last_outbound_at)`,
+      address,
+      domain,
+      at,
+    );
+  }
+}
+
+type SenderHistoryRow = {
+  address: string;
+  message_count: number;
+  outbound_count: number;
+};
+
+export function senderHistory(sql: SqlStorage, addresses: string[]) {
+  const unique = [...new Set(addresses.map((address) => address.toLocaleLowerCase()))].slice(0, 100);
+  const result = new Map<string, SenderHistory>();
+  if (!unique.length) return result;
+  const rows = sql.exec<SenderHistoryRow>(
+    `select address, message_count, outbound_count from senders where address in (${unique.map(() => "?").join(", ")})`,
+    ...unique,
+  ).toArray();
+  for (const row of rows) {
+    result.set(row.address, {
+      inboundCount: Number(row.message_count),
+      outboundCount: Number(row.outbound_count),
+    });
+  }
+  return result;
+}
+
+type BlockRow = { address: string; blocked_at: number };
+
+/** The most specific mailbox block covering an address, if any. */
+export function mailboxBlockFor(sql: SqlStorage, address: string): SenderBlock | null {
+  const patterns = blockPatternsFor(address);
+  if (!patterns.length) return null;
+  const rows = sql.exec<BlockRow>(
+    `select address, blocked_at from senders
+     where blocked_at is not null and address in (${patterns.map(() => "?").join(", ")})`,
+    ...patterns,
+  ).toArray();
+  const pattern = patterns.find((candidate) => rows.some((row) => row.address === candidate));
+  const row = rows.find((candidate) => candidate.address === pattern);
+  return row ? blockFromPattern(row.address, Number(row.blocked_at), false) : null;
+}
+
+export function blockFromPattern(pattern: string, at: number, workspace: boolean): SenderBlock {
+  return { pattern, scope: pattern.startsWith("@") ? "domain" : "address", workspace, at };
+}
+
+/** First workspace-wide pattern (address before domains) that covers an address. */
+export function workspaceBlockFor(address: string, patterns: ReadonlyMap<string, number>) {
+  const pattern = blockPatternsFor(address).find((candidate) => patterns.has(candidate));
+  return pattern ? blockFromPattern(pattern, patterns.get(pattern)!, true) : null;
+}
+
+type StatusRow = { address: string; trusted_at: number | null; reported_at: number | null };
+
+/** Block, trust and report state of each address in this mailbox (lowercased keys). */
+export function senderStatuses(sql: SqlStorage, addresses: string[]) {
+  const unique = [...new Set(addresses.map((address) => address.toLocaleLowerCase()))].slice(0, 100);
+  const result = new Map<string, SenderStatus>();
+  if (!unique.length) return result;
+  const rows = sql.exec<StatusRow>(
+    `select address, trusted_at, reported_at from senders where address in (${unique.map(() => "?").join(", ")})`,
+    ...unique,
+  ).toArray();
+  const byAddress = new Map(rows.map((row) => [row.address, row]));
+  for (const address of unique) {
+    const row = byAddress.get(address);
+    result.set(address, {
+      block: mailboxBlockFor(sql, address),
+      trustedAt: row?.trusted_at == null ? null : Number(row.trusted_at),
+      reportedAt: row?.reported_at == null ? null : Number(row.reported_at),
+    });
+  }
+  return result;
+}
+
+function patternDomain(pattern: string) {
+  return pattern.startsWith("@") ? pattern.slice(1) : pattern.slice(pattern.lastIndexOf("@") + 1);
+}
+
+/** Blocking clears trust: the two decisions are mutually exclusive. */
+export function setSenderBlocked(sql: SqlStorage, pattern: string, blocked: boolean, now: number) {
+  if (!blocked) {
+    sql.exec("update senders set blocked_at = null where address = ?", pattern);
+    return;
+  }
+  sql.exec(
+    `insert into senders (address, domain, blocked_at) values (?, ?, ?)
+     on conflict (address) do update set blocked_at = excluded.blocked_at, trusted_at = null`,
+    pattern,
+    patternDomain(pattern),
+    now,
+  );
+}
+
+/** Trusting an address also lifts a block on that exact address. */
+export function setSenderTrusted(sql: SqlStorage, address: string, trusted: boolean, now: number) {
+  const value = address.toLocaleLowerCase();
+  if (!trusted) {
+    sql.exec("update senders set trusted_at = null where address = ?", value);
+    return;
+  }
+  sql.exec(
+    `insert into senders (address, domain, trusted_at) values (?, ?, ?)
+     on conflict (address) do update set trusted_at = excluded.trusted_at, blocked_at = null`,
+    value,
+    patternDomain(value),
+    now,
+  );
+}
+
+export type BlockedSenderItem = {
+  pattern: string;
+  scope: "address" | "domain";
+  blockedAt: number;
+  messageCount: number;
+};
+
+export function listBlockedSenders(sql: SqlStorage): BlockedSenderItem[] {
+  return sql.exec<{ address: string; blocked_at: number; message_count: number }>(
+    "select address, blocked_at, message_count from senders where blocked_at is not null order by blocked_at desc limit 500",
+  ).toArray().map((row) => ({
+    pattern: row.address,
+    scope: row.address.startsWith("@") ? "domain" : "address",
+    blockedAt: Number(row.blocked_at),
+    messageCount: Number(row.message_count),
+  }));
+}
+
+export function recordSenderFeedback(sql: SqlStorage, input: {
+  id: string;
+  address: string;
+  emailId: string | null;
+  label: FeedbackLabel;
+  userId: string | null;
+  now: number;
+}) {
+  const address = input.address.toLocaleLowerCase();
+  const domain = patternDomain(address);
+  sql.exec(
+    `insert into sender_feedback (id, address, domain, email_id, label, user_id, created_at)
+     values (?, ?, ?, ?, ?, ?, ?)`,
+    input.id,
+    address,
+    domain,
+    input.emailId,
+    input.label,
+    input.userId,
+    input.now,
+  );
+  if (input.label === "not_spam") return;
+  sql.exec(
+    `insert into senders (address, domain, reported_at) values (?, ?, ?)
+     on conflict (address) do update set reported_at = excluded.reported_at, trusted_at = null`,
+    address,
+    domain,
+    input.now,
+  );
+}
+
+/** Report history for the classifier: what people in this mailbox said about the sender before. */
+export function senderFeedbackSummary(sql: SqlStorage, address: string) {
+  const value = address.toLocaleLowerCase();
+  const domain = patternDomain(value);
+  const rows = sql.exec<{ label: FeedbackLabel; same_address: number; count: number }>(
+    `select label, address = ? as same_address, count(*) as count
+     from sender_feedback where domain = ? group by label, address = ?`,
+    value,
+    domain,
+    value,
+  ).toArray();
+  const count = (label: FeedbackLabel, sameAddress: boolean) => rows
+    .filter((row) => row.label === label && (!sameAddress || Boolean(row.same_address)))
+    .reduce((total, row) => total + Number(row.count), 0);
+  return {
+    spamReports: count("spam", true),
+    phishingReports: count("phishing", true),
+    notSpam: count("not_spam", true),
+    domainReports: count("spam", false) + count("phishing", false),
+  };
+}
+
+export type SenderFeedbackSummary = ReturnType<typeof senderFeedbackSummary>;