@gmickel/gno 1.27.0 → 1.28.0

package/src/converters/adapters/email/mime.ts

@@ -0,0 +1,454 @@
+import type {
+  RecordAttachmentInventoryItem,
+  RecordMetadata,
+} from "../../types";
+
+import { RECORD_METADATA_LIMITS } from "../../types";
+import { sanitizeHtmlToText } from "./html";
+import { parseParameterizedHeader } from "./parameters";
+
+const MAX_MIME_DEPTH = 12;
+const MAX_MIME_PARTS = 256;
+const MAX_HEADER_CHARS = 256 * 1024;
+const MAX_HEADER_LINES = 2_048;
+const CONTROL_CHAR_PATTERN = new RegExp(
+  `[${String.fromCharCode(0)}-${String.fromCharCode(31)}${String.fromCharCode(127)}]`,
+  "g"
+);
+const MESSAGE_ID_PATTERN = /<([^<>\s]+)>/g;
+
+export type MailParseErrorKind = "limit" | "malformed";
+
+export class MailParseError extends Error {
+  constructor(
+    message: string,
+    readonly kind: MailParseErrorKind = "malformed"
+  ) {
+    super(message);
+    this.name = "MailParseError";
+  }
+}
+
+export interface ParsedAttachment extends RecordAttachmentInventoryItem {
+  sha256: string;
+}
+
+export interface ParsedEmail {
+  subject?: string;
+  author?: string;
+  participants: string[];
+  sentAt?: string;
+  messageId?: string;
+  inReplyTo?: string;
+  references: string[];
+  body: string;
+  attachments: ParsedAttachment[];
+  metadata: RecordMetadata;
+}
+
+export interface ParseEmailLimits {
+  maxBodyChars: number;
+  maxMetadataChars: number;
+  maxAttachmentBytes: number;
+}
+
+interface MimeState {
+  plainBodies: string[];
+  htmlBodies: string[];
+  attachments: ParsedAttachment[];
+  partCount: number;
+  limits: ParseEmailLimits;
+}
+
+const binaryToBytes = (value: string): Uint8Array => {
+  const bytes = new Uint8Array(value.length);
+  for (let index = 0; index < value.length; index += 1) {
+    bytes[index] = value.charCodeAt(index) & 0xff;
+  }
+  return bytes;
+};
+
+const decodeBytes = (bytes: Uint8Array, charset = "utf-8"): string => {
+  const normalized = charset.trim().toLowerCase().replaceAll("_", "-");
+  const labels: Record<string, string> = {
+    ascii: "windows-1252",
+    "iso-8859-1": "windows-1252",
+    latin1: "windows-1252",
+    "us-ascii": "windows-1252",
+    utf8: "utf-8",
+  };
+  try {
+    return new TextDecoder(labels[normalized] ?? normalized, {
+      fatal: false,
+    }).decode(bytes);
+  } catch {
+    return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
+  }
+};
+
+const decodeQuotedPrintable = (value: string): Uint8Array => {
+  const unfolded = value.replace(/=\r?\n/g, "");
+  const output: number[] = [];
+  for (let index = 0; index < unfolded.length; index += 1) {
+    if (
+      unfolded[index] === "=" &&
+      /^[\da-f]{2}$/i.test(unfolded.slice(index + 1, index + 3))
+    ) {
+      output.push(Number.parseInt(unfolded.slice(index + 1, index + 3), 16));
+      index += 2;
+    } else {
+      output.push(unfolded.charCodeAt(index) & 0xff);
+    }
+  }
+  return Uint8Array.from(output);
+};
+
+const decodeBase64 = (value: string): Uint8Array => {
+  const compact = value.replace(/\s/g, "");
+  if (
+    compact.length === 0 ||
+    compact.length % 4 === 1 ||
+    !/^[a-z\d+/]*={0,2}$/i.test(compact)
+  ) {
+    throw new MailParseError("Malformed base64 MIME body.");
+  }
+  try {
+    return Uint8Array.fromBase64(compact);
+  } catch {
+    throw new MailParseError("Malformed base64 MIME body.");
+  }
+};
+
+const decodeTransfer = (
+  value: string,
+  encoding: string | undefined,
+  maxBytes: number
+): Uint8Array => {
+  const normalized = encoding?.trim().toLowerCase();
+  let bytes: Uint8Array;
+  if (normalized === "base64") {
+    if (value.replace(/\s/g, "").length > maxBytes * 2) {
+      throw new MailParseError(
+        "MIME body exceeds its decoded byte limit.",
+        "limit"
+      );
+    }
+    bytes = decodeBase64(value);
+  } else if (normalized === "quoted-printable") {
+    if (value.length > maxBytes * 3) {
+      throw new MailParseError(
+        "MIME body exceeds its decoded byte limit.",
+        "limit"
+      );
+    }
+    bytes = decodeQuotedPrintable(value);
+  } else {
+    bytes = binaryToBytes(value);
+  }
+  if (bytes.byteLength > maxBytes) {
+    throw new MailParseError(
+      "MIME body exceeds its decoded byte limit.",
+      "limit"
+    );
+  }
+  return bytes;
+};
+
+const decodeHeaderWords = (value: string): string => {
+  const decoded = decodeBytes(binaryToBytes(value)).replace(
+    /(\?=)\s+(=\?)/g,
+    "$1$2"
+  );
+  return decoded
+    .replace(
+      /=\?([^?]+)\?([bq])\?([^?]*)\?=/gi,
+      (_match, charset: string, encoding: string, payload: string) => {
+        try {
+          const bytes =
+            encoding.toLowerCase() === "b"
+              ? decodeBase64(payload)
+              : decodeQuotedPrintable(payload.replaceAll("_", " "));
+          return decodeBytes(bytes, charset);
+        } catch {
+          return "";
+        }
+      }
+    )
+    .replace(/\s+/g, " ")
+    .trim();
+};
+
+const splitHeaderBody = (raw: string): [string, string] => {
+  const match = /\r?\n\r?\n/.exec(raw);
+  if (!match || match.index === undefined) {
+    return [raw, ""];
+  }
+  return [raw.slice(0, match.index), raw.slice(match.index + match[0].length)];
+};
+
+const parseHeaders = (raw: string): Map<string, string> => {
+  if (raw.length > MAX_HEADER_CHARS) {
+    throw new MailParseError(
+      "Mail headers exceed their character limit.",
+      "limit"
+    );
+  }
+  const unfolded = raw.replace(/\r?\n[ \t]+/g, " ");
+  const lines = unfolded.split(/\r?\n/);
+  if (lines.length > MAX_HEADER_LINES) {
+    throw new MailParseError("Mail headers exceed their line limit.", "limit");
+  }
+  const headers = new Map<string, string>();
+  for (const line of lines) {
+    if (!line) continue;
+    const separator = line.indexOf(":");
+    if (separator <= 0) continue;
+    const name = line.slice(0, separator).trim().toLowerCase();
+    const value = line.slice(separator + 1).trim();
+    if (!/^[a-z\d-]+$/.test(name)) continue;
+    const previous = headers.get(name);
+    headers.set(name, previous ? `${previous}, ${value}` : value);
+  }
+  return headers;
+};
+
+const splitMultipart = (body: string, boundary: string): string[] => {
+  if (!boundary || boundary.length > 200 || /[\r\n]/.test(boundary)) {
+    throw new MailParseError("Invalid MIME multipart boundary.");
+  }
+  const marker = `--${boundary}`;
+  const closing = `${marker}--`;
+  const parts: string[] = [];
+  let current: string[] | undefined;
+  for (const line of body.split("\n")) {
+    const normalized = line.endsWith("\r") ? line.slice(0, -1) : line;
+    const boundaryLine = normalized.replace(/[ \t]+$/, "");
+    if (boundaryLine === marker || boundaryLine === closing) {
+      if (current) parts.push(current.join("\n"));
+      current = boundaryLine === closing ? undefined : [];
+      if (boundaryLine === closing) break;
+    } else if (current) {
+      current.push(line);
+    }
+  }
+  if (parts.length === 0) {
+    throw new MailParseError("MIME multipart body has no bounded parts.");
+  }
+  return parts;
+};
+
+const hashBytes = (bytes: Uint8Array): string =>
+  new Bun.CryptoHasher("sha256").update(bytes).digest("hex");
+
+const safeFilename = (value: string | undefined, index: number): string => {
+  const normalized = value
+    ?.normalize("NFC")
+    .replace(CONTROL_CHAR_PATTERN, "")
+    .replaceAll("\\", "/")
+    .split("/")
+    .at(-1)
+    ?.trim();
+  return (normalized || `attachment-${index}`).slice(0, 240);
+};
+
+const walkMimePart = (raw: string, state: MimeState, depth: number): void => {
+  if (depth > MAX_MIME_DEPTH) {
+    throw new MailParseError("MIME nesting exceeds its depth limit.", "limit");
+  }
+  state.partCount += 1;
+  if (state.partCount > MAX_MIME_PARTS) {
+    throw new MailParseError("MIME message exceeds its part limit.", "limit");
+  }
+  const [rawHeaders, body] = splitHeaderBody(raw);
+  const headers = parseHeaders(rawHeaders);
+  const contentType = parseParameterizedHeader(
+    headers.get("content-type"),
+    decodeHeaderWords
+  );
+  const disposition = parseParameterizedHeader(
+    headers.get("content-disposition"),
+    decodeHeaderWords
+  );
+  const mime = contentType.value || "text/plain";
+  if (mime.startsWith("multipart/")) {
+    const boundary = contentType.params.boundary;
+    if (!boundary) {
+      throw new MailParseError("MIME multipart boundary is missing.");
+    }
+    for (const part of splitMultipart(body, boundary)) {
+      walkMimePart(part, state, depth + 1);
+    }
+    return;
+  }
+
+  const filename =
+    disposition.params.filename ?? contentType.params.name ?? undefined;
+  const isAttachment =
+    disposition.value === "attachment" ||
+    Boolean(filename) ||
+    (!mime.startsWith("text/") && mime !== "message/rfc822");
+  const bytes = decodeTransfer(
+    body,
+    headers.get("content-transfer-encoding"),
+    isAttachment
+      ? state.limits.maxAttachmentBytes
+      : state.limits.maxBodyChars * 4
+  );
+  if (isAttachment || mime === "message/rfc822") {
+    state.attachments.push({
+      name: safeFilename(filename, state.attachments.length + 1),
+      mime,
+      bytes: bytes.byteLength,
+      disposition: disposition.value === "inline" ? "inline" : "attachment",
+      sha256: hashBytes(bytes),
+    });
+    return;
+  }
+  const charset = contentType.params.charset ?? "utf-8";
+  const decoded = decodeBytes(bytes, charset);
+  if (mime === "text/html") {
+    state.htmlBodies.push(sanitizeHtmlToText(decoded));
+  } else if (mime === "text/plain") {
+    state.plainBodies.push(decoded.trim());
+  }
+};
+
+const normalizeMessageId = (value: string | undefined): string | undefined => {
+  if (!value) return undefined;
+  const match = /<([^<>\s]+)>/.exec(value);
+  const normalized = (match?.[1] ?? value)
+    .normalize("NFC")
+    .replace(CONTROL_CHAR_PATTERN, "")
+    .trim();
+  return normalized
+    ? normalized.slice(0, RECORD_METADATA_LIMITS.maxIdentifierChars)
+    : undefined;
+};
+
+const extractMessageIds = (value: string | undefined): string[] => {
+  if (!value) return [];
+  const ids: string[] = [];
+  for (const match of value.matchAll(MESSAGE_ID_PATTERN)) {
+    if (match[1]) {
+      ids.push(
+        match[1]
+          .normalize("NFC")
+          .slice(0, RECORD_METADATA_LIMITS.maxIdentifierChars)
+      );
+    }
+  }
+  return ids;
+};
+
+const normalizeDate = (value: string | undefined): string | undefined => {
+  if (!value) return undefined;
+  const trimmed = value.trim();
+  const namedZones: Record<string, string> = {
+    CDT: "-0500",
+    CST: "-0600",
+    EDT: "-0400",
+    EST: "-0500",
+    GMT: "+0000",
+    MDT: "-0600",
+    MST: "-0700",
+    PDT: "-0700",
+    PST: "-0800",
+    UT: "+0000",
+    UTC: "+0000",
+  };
+  const namedMatch = /\b([A-Z]{2,3})\s*$/.exec(trimmed);
+  const numericZone = /[+-]\d{4}\s*$/.test(trimmed);
+  const replacement = namedMatch ? namedZones[namedMatch[1] ?? ""] : undefined;
+  if (!(numericZone || replacement)) return undefined;
+  const normalized = replacement
+    ? trimmed.slice(0, namedMatch?.index).trimEnd() + ` ${replacement}`
+    : trimmed;
+  const parsed = new Date(normalized);
+  return Number.isNaN(parsed.getTime()) ? undefined : parsed.toISOString();
+};
+
+const normalizedHeader = (
+  headers: Map<string, string>,
+  name: string
+): string | undefined => {
+  const value = headers.get(name);
+  if (!value) return undefined;
+  const decoded = decodeHeaderWords(value)
+    .replace(CONTROL_CHAR_PATTERN, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+  return decoded ? decoded.slice(0, 8_192) : undefined;
+};
+
+export const parseEmail = (
+  raw: string,
+  limits: ParseEmailLimits
+): ParsedEmail => {
+  const [rawHeaders] = splitHeaderBody(raw);
+  const headers = parseHeaders(rawHeaders);
+  const state: MimeState = {
+    plainBodies: [],
+    htmlBodies: [],
+    attachments: [],
+    partCount: 0,
+    limits,
+  };
+  walkMimePart(raw, state, 0);
+  const bodyParts = state.plainBodies.some(Boolean)
+    ? state.plainBodies
+    : state.htmlBodies;
+  const body = bodyParts.filter(Boolean).join("\n\n").trim();
+  if (
+    headers.size === 0 &&
+    body.length === 0 &&
+    state.attachments.length === 0
+  ) {
+    throw new MailParseError("Mail message has no parseable content.");
+  }
+  if (body.length > limits.maxBodyChars) {
+    throw new MailParseError(
+      "Decoded mail body exceeds its character limit.",
+      "limit"
+    );
+  }
+
+  const author = normalizedHeader(headers, "from");
+  const participantHeaders = ["from", "to", "cc", "bcc"]
+    .map((name) => normalizedHeader(headers, name))
+    .filter((value): value is string => Boolean(value));
+  const participants = [...new Set(participantHeaders)];
+  const messageId = normalizeMessageId(headers.get("message-id"));
+  const inReplyTo = normalizeMessageId(headers.get("in-reply-to"));
+  const references = extractMessageIds(headers.get("references"));
+  const sentAt = normalizeDate(headers.get("date"));
+  const threadId = references[0] ?? inReplyTo ?? messageId;
+  const metadata: RecordMetadata = {
+    author,
+    participants,
+    categories: ["email"],
+    dateFields: sentAt ? { sentAt } : undefined,
+    messageId,
+    inReplyTo,
+    references,
+    threadId,
+    attachments: state.attachments,
+  };
+  if (JSON.stringify(metadata).length > limits.maxMetadataChars) {
+    throw new MailParseError(
+      "Mail metadata exceeds its character limit.",
+      "limit"
+    );
+  }
+  return {
+    subject: normalizedHeader(headers, "subject"),
+    author,
+    participants,
+    sentAt,
+    messageId,
+    inReplyTo,
+    references,
+    body,
+    attachments: state.attachments,
+    metadata,
+  };
+};

package/src/converters/adapters/email/parameters.ts

@@ -0,0 +1,77 @@
+interface HeaderValue {
+  value: string;
+  params: Record<string, string>;
+}
+
+type DecodeHeaderWords = (value: string) => string;
+
+const decodeExtendedParameter = (value: string): string => {
+  const encoded = /^[^']*'[^']*'(.*)$/.exec(value)?.[1] ?? value;
+  try {
+    return decodeURIComponent(encoded);
+  } catch {
+    return encoded;
+  }
+};
+
+export const parseParameterizedHeader = (
+  raw: string | undefined,
+  decodeHeaderWords: DecodeHeaderWords
+): HeaderValue => {
+  if (!raw) return { value: "", params: {} };
+  const pieces = raw.match(/(?:[^;"']+|"[^"]*"|'[^']*')+/g) ?? [raw];
+  const value = pieces[0]?.trim().toLowerCase() ?? "";
+  const params: Record<string, string> = {};
+  const continuations = new Map<
+    string,
+    Array<{ encoded: boolean; index: number; value: string }>
+  >();
+  for (const piece of pieces.slice(1)) {
+    const separator = piece.indexOf("=");
+    if (separator <= 0) continue;
+    const key = piece.slice(0, separator).trim().toLowerCase();
+    let parameter = piece.slice(separator + 1).trim();
+    if (
+      (parameter.startsWith('"') && parameter.endsWith('"')) ||
+      (parameter.startsWith("'") && parameter.endsWith("'"))
+    ) {
+      parameter = parameter.slice(1, -1);
+    }
+    const continuation = /^(.+)\*(\d+)(\*)?$/.exec(key);
+    if (continuation) {
+      const base = continuation[1];
+      const index = Number.parseInt(continuation[2] ?? "", 10);
+      if (base && Number.isSafeInteger(index)) {
+        const segments = continuations.get(base) ?? [];
+        segments.push({
+          encoded: Boolean(continuation[3]),
+          index,
+          value: parameter,
+        });
+        continuations.set(base, segments);
+      }
+      continue;
+    }
+    const decoded = key.endsWith("*")
+      ? decodeExtendedParameter(parameter)
+      : parameter;
+    params[key.replace(/\*$/, "")] = decodeHeaderWords(decoded);
+  }
+  for (const [key, unsorted] of continuations) {
+    const segments = unsorted.toSorted(
+      (left, right) => left.index - right.index
+    );
+    if (
+      segments[0]?.index !== 0 ||
+      segments.some((segment, index) => segment.index !== index)
+    ) {
+      continue;
+    }
+    const joined = segments.map((segment) => segment.value).join("");
+    const decoded = segments.some((segment) => segment.encoded)
+      ? decodeExtendedParameter(joined)
+      : joined;
+    params[key] = decodeHeaderWords(decoded);
+  }
+  return { value, params };
+};