@open-workspace/email 0.0.0-stage → 0.1.0

package/worker/mail/office-conversion.ts

@@ -0,0 +1,38 @@
+import { MAX_OFFICE_PREVIEW_BYTES, officePreviewKey } from "../../shared/office-preview";
+
+export type OfficeConversionResult = "ready" | "failed";
+
+export async function convertOfficeDocument(
+  bucket: R2Bucket,
+  send: (request: Request) => Promise<Response>,
+  sourceKey: string,
+  extension: string,
+): Promise<OfficeConversionResult> {
+  const source = await bucket.get(sourceKey);
+  if (!source) return "failed";
+  // FormData buffers the upload in the Worker; cap it before reading the body.
+  if (source.size > MAX_OFFICE_PREVIEW_BYTES) {
+    await source.body.cancel();
+    return "failed";
+  }
+  const form = new FormData();
+  form.append("files", new Blob([await source.arrayBuffer()]), `document.${extension}`);
+  const response = await send(new Request("http://container/forms/libreoffice/convert", {
+    method: "POST",
+    body: form,
+    signal: AbortSignal.timeout(120_000),
+  }));
+  if (!response.ok) {
+    await response.body?.cancel();
+    if ([400, 403, 415, 422].includes(response.status)) return "failed";
+    throw new Error(`Office conversion failed with status ${response.status}`);
+  }
+  if (!response.body || response.headers.get("content-type")?.split(";", 1)[0] !== "application/pdf") {
+    await response.body?.cancel();
+    throw new Error("Office converter did not return a PDF");
+  }
+  await bucket.put(officePreviewKey(sourceKey), response.body, {
+    httpMetadata: { contentType: "application/pdf" },
+  });
+  return "ready";
+}

package/worker/mail/office-preview-container.ts

@@ -0,0 +1,57 @@
+import { DurableObject } from "cloudflare:workers";
+import { officePreviewExtension, officePreviewKey } from "../../shared/office-preview";
+import { convertOfficeDocument, type OfficeConversionResult } from "./office-conversion";
+
+export class OfficePreviewContainer extends DurableObject<Env> {
+  private conversion: Promise<OfficeConversionResult> | null = null;
+
+  async convert(sourceKey: string, extension: string) {
+    if (officePreviewExtension(`document.${extension}`) !== extension) return "failed" as const;
+    // Queue redelivery can overlap. Each attachment uses its own Durable Object,
+    // so duplicate requests share one conversion and cannot stop each other's container.
+    if (this.conversion) return this.conversion;
+    this.conversion = this.runConversion(sourceKey, extension);
+    try {
+      return await this.conversion;
+    } finally {
+      this.conversion = null;
+    }
+  }
+
+  private async runConversion(sourceKey: string, extension: string): Promise<OfficeConversionResult> {
+    if (await this.env.MAIL_STORAGE.head(officePreviewKey(sourceKey))) return "ready";
+    const container = this.ctx.container;
+    if (!container) throw new Error("Office preview container is not configured");
+    try {
+      if (!container.running) container.start({ enableInternet: false });
+      await container.setInactivityTimeout(10_000);
+      const port = container.getTcpPort(3000);
+      await waitForConverter(port);
+      return await convertOfficeDocument(
+        this.env.MAIL_STORAGE,
+        (request) => port.fetch(request),
+        sourceKey,
+        extension,
+      );
+    } finally {
+      // R2 has consumed the PDF before this runs; no container is kept warm.
+      await container.destroy();
+    }
+  }
+}
+
+async function waitForConverter(port: Fetcher) {
+  for (let attempt = 0; attempt < 120; attempt += 1) {
+    try {
+      const response = await port.fetch("http://container/health", {
+        signal: AbortSignal.timeout(1_000),
+      });
+      await response.body?.cancel();
+      if (response.ok) return;
+    } catch {
+      // The container process starts before its HTTP port is ready.
+    }
+    await scheduler.wait(500);
+  }
+  throw new Error("Office preview container did not become ready");
+}

package/worker/mail/office-preview-queue.ts

@@ -0,0 +1,49 @@
+import { z } from "zod";
+import { officePreviewExtension } from "../../shared/office-preview";
+import { mailboxStub } from "../mailbox";
+
+const jobSchema = z.object({
+  mailboxId: z.string().min(1),
+  messageId: z.string().min(1),
+  attachmentId: z.string().min(1),
+  sourceKey: z.string().min(1),
+});
+
+export async function consumeOfficePreviews(batch: MessageBatch<unknown>, env: Env) {
+  for (const message of batch.messages) {
+    const parsed = jobSchema.safeParse(message.body);
+    if (!parsed.success) {
+      message.ack();
+      continue;
+    }
+    const job = parsed.data;
+    const mailbox = mailboxStub(env, job.mailboxId);
+    try {
+      const file = await mailbox.getAttachment(job.messageId, job.attachmentId);
+      if (!file || file.r2Key !== job.sourceKey) {
+        // A previous attempt may have produced a PDF after the email was deleted.
+        await mailbox.completeOfficePreview(job.messageId, job.attachmentId, job.sourceKey, "failed");
+        message.ack();
+        continue;
+      }
+      if (file.pdfPreviewStatus !== "pending") {
+        message.ack();
+        continue;
+      }
+      const extension = officePreviewExtension(file.filename);
+      const status = extension
+        ? await env.OFFICE_PREVIEW_CONTAINER.getByName(job.sourceKey).convert(job.sourceKey, extension)
+        : "failed";
+      await mailbox.completeOfficePreview(job.messageId, job.attachmentId, job.sourceKey, status);
+      message.ack();
+    } catch (error) {
+      console.error("Could not generate Office preview", { messageId: job.messageId, error });
+      if (message.attempts < 3) {
+        message.retry({ delaySeconds: 30 * message.attempts });
+        continue;
+      }
+      await mailbox.completeOfficePreview(job.messageId, job.attachmentId, job.sourceKey, "failed");
+      message.ack();
+    }
+  }
+}

package/worker/mail/org-domain.ts

@@ -0,0 +1,15 @@
+import { getDomain } from "tldts";
+import { addressDomain } from "../../shared/mail-headers";
+
+/** Organizational domain (Public Suffix List registrable domain) of a host or address. */
+export function organizationalDomain(value: string | null | undefined) {
+  const domain = addressDomain(value);
+  if (!domain) return null;
+  return getDomain(domain) ?? domain;
+}
+
+/** DMARC relaxed alignment: both sides share one organizational domain. */
+export function relaxedAligned(left: string | null | undefined, right: string | null | undefined) {
+  const a = organizationalDomain(left);
+  return Boolean(a) && a === organizationalDomain(right);
+}

package/worker/mail/outbound-content.ts

@@ -0,0 +1,208 @@
+import type { Email } from "../mailbox/schema";
+import {
+  LINKED_ATTACHMENT_TEXT_PREFIX,
+  linkedAttachmentTextToken,
+} from "../../shared/mail";
+
+const MAX_CONTEXT_CHARACTERS = 24_000;
+
+type ContentSource = Pick<
+  Email,
+  "fromJson" | "toJson" | "ccJson" | "subject" | "timelineAt" | "bodyText" | "preview"
+>;
+
+export function escapeHtml(value: string) {
+  return value
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;")
+    .replaceAll("'", "&#039;");
+}
+
+function displayAddress(email: ContentSource) {
+  const from = email.fromJson[0];
+  if (!from) return "Unknown sender";
+  return from.name ? `${from.name} <${from.address}>` : from.address;
+}
+
+function clippedBody(email: ContentSource) {
+  const value = email.bodyText?.trim() || email.preview.trim() || "No text body";
+  if (value.length <= MAX_CONTEXT_CHARACTERS) return value;
+  return `${value.slice(0, MAX_CONTEXT_CHARACTERS).trimEnd()}\n[Context truncated]`;
+}
+
+function metadataLines(email: ContentSource) {
+  return [
+    `From: ${displayAddress(email)}`,
+    `Date: ${email.timelineAt.toISOString()}`,
+    `Subject: ${email.subject}`,
+    ...(email.toJson.length
+      ? [`To: ${email.toJson.map((item) => item.address).join(", ")}`]
+      : []),
+    ...(email.ccJson.length
+      ? [`Cc: ${email.ccJson.map((item) => item.address).join(", ")}`]
+      : []),
+  ];
+}
+
+export function contextForNewRecipient(email: ContentSource) {
+  return [...metadataLines(email), "", clippedBody(email)].join("\n");
+}
+
+export function forwardedMessageText(email: ContentSource) {
+  return ["Forwarded message", ...metadataLines(email), "", clippedBody(email)].join("\n");
+}
+
+export function forwardedMessageHtml(email: ContentSource) {
+  const metadata = metadataLines(email)
+    .map((line) => `<div>${escapeHtml(line)}</div>`)
+    .join("");
+  return `<section data-forwarded-message><strong>Forwarded message</strong>${metadata}<pre style="white-space:pre-wrap">${escapeHtml(clippedBody(email))}</pre></section>`;
+}
+
+export function appendForwardedMessage(
+  bodyText: string,
+  bodyHtml: string | undefined,
+  forwarded: ContentSource | null,
+) {
+  if (!forwarded) return { bodyText, bodyHtml };
+  const text = forwardedMessageText(forwarded);
+  const nextText = `${bodyText.trimEnd()}${bodyText.trimEnd() ? "\n\n" : ""}${text}`;
+  return {
+    bodyText: nextText,
+    bodyHtml: bodyHtml
+      ? `${bodyHtml}${forwardedMessageHtml(forwarded)}`
+      : undefined,
+  };
+}
+
+export function textWithQuotedContext(bodyText: string, quotedText: string | null) {
+  if (!quotedText) return bodyText;
+  const quote = quotedText.split("\n").map((line) => `> ${line}`).join("\n");
+  return `${bodyText.trimEnd()}${bodyText.trimEnd() ? "\n\n" : ""}${quote}`;
+}
+
+export function htmlWithQuotedContext(bodyHtml: string, quotedText: string | null) {
+  if (!quotedText) return bodyHtml;
+  return `${bodyHtml}<blockquote><pre style="white-space:pre-wrap">${escapeHtml(quotedText)}</pre></blockquote>`;
+}
+
+export function externalizeLinkedInlineImages(
+  bodyHtml: string,
+  attachments: Array<{
+    disposition: "attachment" | "inline";
+    contentId: string | null;
+    filename: string;
+    downloadUrl: string;
+  }>,
+) {
+  let nextHtml = bodyHtml;
+  for (const attachment of attachments) {
+    if (attachment.disposition !== "inline" || !attachment.contentId) continue;
+    const escapedContentId = attachment.contentId.replace(
+      /[.*+?^${}()|[\]\\]/gu,
+      "\\$&",
+    );
+    const image = new RegExp(
+      `<img(?=[^>]*\\bsrc=(["'])cid:${escapedContentId}\\1)[^>]*>`,
+      "giu",
+    );
+    nextHtml = nextHtml.replace(
+      image,
+      `<a href="${escapeHtml(attachment.downloadUrl)}">${escapeHtml(attachment.filename)}</a>`,
+    );
+  }
+  return nextHtml;
+}
+
+type PositionedLinkedAttachment = {
+  uploadId: string | null;
+  filename: string;
+  downloadUrl: string | null;
+};
+
+function escapeRegExp(value: string) {
+  return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
+}
+
+const unresolvedLinkedAttachmentText = new RegExp(
+  `\\[\\[${escapeRegExp(LINKED_ATTACHMENT_TEXT_PREFIX)}[^\\]]+\\]\\]`,
+  "gu",
+);
+const unresolvedLinkedAttachmentHtml = new RegExp(
+  `<span(?=[^>]*\\bdata-linked-attachment=(["'])[^"']*\\1)[^>]*>[\\s\\S]*?<\\/span>`,
+  "giu",
+);
+
+function linkedAttachmentHtml(filename: string, downloadUrl: string) {
+  return [
+    '<span data-linked-attachment-card style="display:block;margin:12px 0">',
+    `<a href="${escapeHtml(downloadUrl)}" style="display:inline-block;box-sizing:border-box;max-width:100%;padding:10px 12px;border:1px solid #dadce0;border-radius:8px;color:#1967d2;text-decoration:none;font-family:Arial,sans-serif">`,
+    `<strong style="color:#202124">${escapeHtml(filename)}</strong>`,
+    '<span style="margin-left:8px;font-size:12px;color:#5f6368">Download file · link expires in 30 days</span>',
+    "</a>",
+    "</span>",
+  ].join("");
+}
+
+/**
+ * Download URLs only exist once the outgoing message has an id and token.
+ * Composer nodes therefore carry an upload-id placeholder that is resolved
+ * here, while preserving the position chosen in the editor.
+ */
+export function resolveLinkedAttachmentPlaceholders(input: {
+  bodyText: string;
+  bodyHtml: string | undefined;
+  attachments: PositionedLinkedAttachment[];
+}) {
+  let bodyText = input.bodyText;
+  let bodyHtml = input.bodyHtml;
+  const textPlacedUploadIds = new Set<string>();
+  const htmlPlacedUploadIds = new Set<string>();
+
+  for (const attachment of input.attachments) {
+    if (!attachment.uploadId) continue;
+    const textToken = linkedAttachmentTextToken(attachment.uploadId);
+    if (bodyText.includes(textToken)) {
+      bodyText = bodyText.replaceAll(
+        textToken,
+        attachment.downloadUrl
+          ? `${attachment.filename}: ${attachment.downloadUrl}`
+          : "",
+      );
+      if (attachment.downloadUrl) {
+        textPlacedUploadIds.add(attachment.uploadId);
+      }
+    }
+    if (!bodyHtml) continue;
+
+    const marker = escapeRegExp(attachment.uploadId);
+    const placeholder = new RegExp(
+      `<span(?=[^>]*\\bdata-linked-attachment=(["'])${marker}\\1)[^>]*>[\\s\\S]*?<\\/span>`,
+      "giu",
+    );
+    let matched = false;
+    bodyHtml = bodyHtml.replace(placeholder, () => {
+      matched = true;
+      return attachment.downloadUrl
+        ? linkedAttachmentHtml(attachment.filename, attachment.downloadUrl)
+        : "";
+    });
+    if (matched && attachment.downloadUrl) {
+      htmlPlacedUploadIds.add(attachment.uploadId);
+    }
+  }
+
+  bodyText = bodyText.replace(unresolvedLinkedAttachmentText, "");
+  if (bodyHtml) {
+    bodyHtml = bodyHtml.replace(unresolvedLinkedAttachmentHtml, "");
+  }
+
+  return {
+    bodyText: bodyText.replace(/\n{3,}/gu, "\n\n"),
+    bodyHtml,
+    textPlacedUploadIds,
+    htmlPlacedUploadIds,
+  };
+}

package/worker/mail/outbound-delivery.ts

@@ -0,0 +1,83 @@
+import type { Email } from "../mailbox/schema";
+import {
+  htmlWithQuotedContext,
+  textWithQuotedContext,
+} from "./outbound-content";
+import { dedupeRecipientFields } from "./recipients";
+
+export function emailDestinations(
+  message: Pick<Email, "toJson" | "ccJson" | "bccJson">,
+) {
+  const { to, cc, bcc } = dedupeRecipientFields({
+    to: message.toJson.map((item) => item.address),
+    cc: message.ccJson.map((item) => item.address),
+    bcc: message.bccJson.map((item) => item.address),
+  });
+  if (to.length) {
+    return {
+      to,
+      cc: cc.length ? cc : undefined,
+      bcc: bcc.length ? bcc : undefined,
+    };
+  }
+  if (cc.length) return { cc, bcc: bcc.length ? bcc : undefined };
+  if (bcc.length) return { bcc };
+  throw new Error("Queued message has no recipients");
+}
+
+export async function prepareOutboundDelivery(env: Env, message: Email) {
+  const storedHtml = message.bodyHtmlR2Key
+    ? await env.MAIL_STORAGE.get(message.bodyHtmlR2Key).then(
+        (object) => object?.text(),
+      )
+    : undefined;
+  const html = storedHtml
+    ? htmlWithQuotedContext(storedHtml, message.quotedText)
+    : undefined;
+  const attachments: EmailAttachment[] = await Promise.all(
+    message.attachmentsJson
+      .filter((attachment) => attachment.delivery === "attached")
+      .map(async (attachment) => {
+        const object = await env.MAIL_STORAGE.get(attachment.r2Key);
+        if (!object) throw new Error(`Attachment ${attachment.filename} is missing`);
+        const content = await object.arrayBuffer();
+        return attachment.disposition === "inline"
+          ? {
+              disposition: "inline" as const,
+              contentId: attachment.contentId ?? attachment.id,
+              filename: attachment.filename,
+              type: attachment.contentType,
+              content,
+            }
+          : {
+              disposition: "attachment" as const,
+              filename: attachment.filename,
+              type: attachment.contentType,
+              content,
+            };
+      }),
+  );
+  const from = message.fromJson[0];
+  if (!from) throw new Error("Outgoing message has no From address");
+
+  return {
+    from: { name: from.name ?? "", email: from.address },
+    ...emailDestinations(message),
+    replyTo: message.replyToJson[0]?.address,
+    subject: message.subject,
+    text: textWithQuotedContext(
+      message.bodyText ?? "",
+      message.quotedText,
+    ) || undefined,
+    html,
+    attachments: attachments.length ? attachments : undefined,
+    headers: {
+      ...(message.inReplyToJson.length
+        ? { "In-Reply-To": message.inReplyToJson.join(" ") }
+        : {}),
+      ...(message.referencesJson.length
+        ? { References: message.referencesJson.join(" ") }
+        : {}),
+    },
+  };
+}

package/worker/mail/outbound-service.ts

@@ -0,0 +1,145 @@
+import { mailboxStub } from "../mailbox";
+import type { Email } from "../mailbox/schema";
+import {
+  discardPreparedObjects,
+  messageIdForRequest,
+  outgoingRequestFingerprint,
+  prepareOutgoingEmail,
+  type OutgoingMessageInput,
+} from "./outbound";
+import { discardComposerUploads } from "./uploads";
+import {
+  deferWebhookTask,
+  emailWebhookEventId,
+  queueWebhookEvent,
+} from "@open-workspace/core/worker/webhooks/service.ts";
+
+export class OutgoingRequestConflictError extends Error {}
+
+function deferComposerUploadCleanup(input: {
+  env: Env;
+  compose: OutgoingMessageInput;
+  uploadIds: string[];
+  defer: (task: Promise<unknown>) => void;
+}) {
+  const uploadIds = [...new Set(input.uploadIds)];
+  if (!uploadIds.length) return;
+  input.defer(discardComposerUploads({
+    env: input.env,
+    mailboxId: input.compose.mailboxId,
+    userId: input.compose.userId,
+    uploadIds,
+  }));
+}
+
+export function deferEmailSentWebhook(input: {
+  env: Env;
+  mailboxId: string;
+  email: Pick<Email, "id" | "timelineAt" | "transportState">;
+  defer: (task: Promise<unknown>) => void;
+}) {
+  if (input.email.transportState !== "submitted") return;
+  deferWebhookTask(input.defer, () => queueWebhookEvent(input.env, {
+    eventId: emailWebhookEventId(
+      "email.sent",
+      input.mailboxId,
+      input.email.id,
+    ),
+    eventType: "email.sent",
+    occurredAt: input.email.timelineAt.getTime(),
+    source: {
+      kind: "email",
+      mailboxId: input.mailboxId,
+      messageId: input.email.id,
+    },
+  }));
+}
+
+export async function submitOutgoing(input: {
+  env: Env;
+  requestUrl: string;
+  fromAddress: string;
+  fromName: string;
+  compose: OutgoingMessageInput;
+  conversationId: string;
+  related: Email | null;
+  forwarded: Email | null;
+  includeRelatedContext: boolean;
+  defer: (task: Promise<unknown>) => void;
+}) {
+  const { compose } = input;
+  const stub = mailboxStub(input.env, compose.mailboxId);
+  const id = await messageIdForRequest(compose.mailboxId, compose.requestId);
+  const requestFingerprint = await outgoingRequestFingerprint(compose);
+  const existing = await stub.getEmail(id);
+  if (existing) {
+    if (
+      existing.direction !== "outgoing"
+      || existing.requestFingerprint !== requestFingerprint
+    ) {
+      // The conflicting request was not accepted, so its composer still owns
+      // these immutable uploads and may retry them with a fresh attempt id.
+      throw new OutgoingRequestConflictError(
+        "Request identifier is already in use with different content",
+      );
+    }
+    deferComposerUploadCleanup({
+      ...input,
+      uploadIds: compose.attachments.map((attachment) => attachment.uploadId),
+    });
+    deferEmailSentWebhook({
+      env: input.env,
+      mailboxId: compose.mailboxId,
+      email: existing,
+      defer: input.defer,
+    });
+    return { email: existing, inserted: false };
+  }
+
+  const prepared = await prepareOutgoingEmail({
+    env: input.env,
+    requestUrl: input.requestUrl,
+    compose,
+    requestFingerprint,
+    id,
+    conversationId: input.conversationId,
+    related: input.related,
+    forwarded: input.forwarded,
+    includeRelatedContext: input.includeRelatedContext,
+    fromAddress: input.fromAddress,
+    fromName: input.fromName,
+    now: new Date(),
+  });
+
+  let submission: Awaited<ReturnType<typeof stub.submitOutgoing>>;
+  try {
+    submission = await stub.submitOutgoing(prepared.email);
+  } catch (error) {
+    await discardPreparedObjects(input.env, prepared.storageKeys);
+    throw error;
+  }
+  if (submission.outcome !== "inserted") {
+    await discardPreparedObjects(input.env, prepared.storageKeys);
+    if (submission.outcome === "conflict") {
+      // Do not clean the losing request's composer uploads. Only prepared
+      // message-scoped copies belong to this failed submission attempt.
+      throw new OutgoingRequestConflictError(
+        "Request identifier is already in use with different content",
+      );
+    }
+  }
+  deferComposerUploadCleanup({
+    ...input,
+    uploadIds: prepared.composerUploadIds,
+  });
+  deferEmailSentWebhook({
+    env: input.env,
+    mailboxId: compose.mailboxId,
+    email: submission.email,
+    defer: input.defer,
+  });
+  return {
+    email: submission.email,
+    inserted: submission.outcome === "inserted",
+  };
+}

package/worker/mail/outbound-threading.ts

@@ -0,0 +1,70 @@
+import type { ReplyPlan } from "../../shared/mail";
+import type { Email } from "../mailbox/schema";
+import {
+  normalizedParticipantSet,
+  sameParticipantSet,
+} from "./participants";
+
+type ParticipantEmail = Pick<
+  Email,
+  "fromJson" | "toJson" | "ccJson" | "bccJson" | "listPostAddress"
+>;
+
+function normalizedRecipients(values: string[], ownAddress: string) {
+  return normalizedParticipantSet(values, ownAddress);
+}
+
+/**
+ * RFC headers may still reference the source message, but a one-person answer
+ * from a visible group is a separate local chat. Matching List-Post is the one
+ * intentional single-address exception because that address represents a list.
+ */
+export function shouldDetachOutboundReply(input: {
+  ownAddress: string;
+  plan: Pick<ReplyPlan, "isGroup" | "actions">;
+  to: string[];
+  cc: string[];
+}) {
+  if (!input.plan.isGroup) return false;
+
+  const actual = normalizedRecipients([...input.to, ...input.cc], input.ownAddress);
+  if (actual.size > 1) return false;
+
+  const listAction = input.plan.actions.find((action) => action.mode === "reply_list");
+  const matchesListPost = listAction
+    ? sameParticipantSet(
+        actual,
+        normalizedRecipients([...listAction.to, ...listAction.cc], input.ownAddress),
+      )
+    : false;
+  return !matchesListPost;
+}
+
+function previousParticipants(ownAddress: string, history: ParticipantEmail[]) {
+  const values = history.flatMap((message) => [
+    ...message.fromJson,
+    ...message.toJson,
+    ...message.ccJson,
+    ...message.bccJson,
+    ...(message.listPostAddress
+      ? [{ address: message.listPostAddress, name: null }]
+      : []),
+  ]).map((item) => item.address);
+  return normalizedRecipients(values, ownAddress);
+}
+
+export function hasNewRecipients(input: {
+  ownAddress: string;
+  history: ParticipantEmail[];
+  to: string[];
+  cc: string[];
+  bcc: string[];
+}) {
+  if (!input.history.length) return false;
+  const previous = previousParticipants(input.ownAddress, input.history);
+  const next = normalizedRecipients(
+    [...input.to, ...input.cc, ...input.bcc],
+    input.ownAddress,
+  );
+  return [...next].some((recipient) => !previous.has(recipient));
+}