@procedures/mail 0.0.0-stage → 0.2.3

package/src/mime.ts

@@ -0,0 +1,160 @@
+/**
+ * The headers and the readable text of a raw RFC 822 message. It reads multipart bodies,
+ * quoted-printable and base64, and HTML-only messages. It is not a mail client: it ignores
+ * attachments, inline images and the headers of the parts.
+ */
+
+interface Parsed {
+  readonly headers: ReadonlyMap<string, string>;
+  /** The body, still in its transfer encoding, one character for each byte */
+  readonly body: string;
+}
+
+const split = (text: string): Parsed => {
+  const at = text.search(/\r?\n\r?\n/);
+  const head = at === -1 ? text : text.slice(0, at);
+  const body = at === -1 ? "" : text.slice(at).replace(/^\r?\n\r?\n/, "");
+  const headers = new Map<string, string>();
+  // A header continues on each line that starts with white space (RFC 5322 folding)
+  for (const line of head.replace(/\r?\n[ \t]+/g, " ").split(/\r?\n/)) {
+    const colon = line.indexOf(":");
+    if (colon <= 0) continue;
+    headers.set(line.slice(0, colon).trim().toLowerCase(), line.slice(colon + 1).trim());
+  }
+  return { headers, body };
+};
+
+const param = (value: string, name: string): string | undefined => {
+  const match = new RegExp(`(?:^|;)\\s*${name}\\s*=\\s*(?:"([^"]*)"|([^;\\s]+))`, "i").exec(value);
+  return match?.[1] ?? match?.[2];
+};
+
+const quotedPrintable = (text: string): string =>
+  text.replace(/=\r?\n/g, "").replace(/=([0-9A-F]{2})/gi, (_, hex: string) => String.fromCharCode(Number.parseInt(hex, 16)));
+
+const decode = (body: string, encoding: string | undefined, charset: string | undefined): string => {
+  const transfer = (encoding ?? "7bit").toLowerCase();
+  const bytes =
+    transfer === "base64"
+      ? Buffer.from(body.replace(/\s+/g, ""), "base64")
+      : transfer === "quoted-printable"
+        ? Buffer.from(quotedPrintable(body), "latin1")
+        : Buffer.from(body, "latin1");
+  const set = (charset ?? "utf-8").toLowerCase();
+  return set === "utf-8" || set === "utf8" || set === "us-ascii" ? bytes.toString("utf8") : bytes.toString("latin1");
+};
+
+const ENTITIES: Readonly<Record<string, string>> = {
+  nbsp: " ",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  apos: "'",
+  ndash: "-",
+  mdash: "-",
+  rsquo: "'",
+  lsquo: "'",
+  ldquo: '"',
+  rdquo: '"',
+};
+
+/** `&amp;` is the last replacement, so `&amp;lt;` stays the text "&lt;". */
+const entities = (text: string): string =>
+  text
+    .replace(/&#(\d+);/g, (_, code: string) => String.fromCodePoint(Number(code)))
+    .replace(/&#x([0-9a-f]+);/gi, (_, hex: string) => String.fromCodePoint(Number.parseInt(hex, 16)))
+    .replace(/&([a-z]+);/gi, (whole, name: string) => ENTITIES[name.toLowerCase()] ?? whole)
+    .replace(/&amp;/gi, "&");
+
+/** This function gives the text of an HTML body as lines. A cell, a row, a paragraph or a break ends a line. */
+export const htmlText = (html: string): string =>
+  entities(
+    html
+      .replace(/<(script|style|head)\b[\s\S]*?<\/\1>/gi, " ")
+      .replace(/<br\s*\/?>/gi, "\n")
+      .replace(/<\/(?:p|div|tr|td|th|li|h[1-6]|table|tbody|section)>/gi, "\n")
+      .replace(/<[^>]+>/g, " ")
+  )
+    .replace(/[ \t ]+/g, " ")
+    .replace(/ *\n */g, "\n")
+    .replace(/\n{2,}/g, "\n")
+    .trim();
+
+const walk = (text: string, plain: string[], html: string[], depth: number): void => {
+  if (depth > 8) return;
+  const { headers, body } = split(text);
+  const type = headers.get("content-type") ?? "text/plain";
+  const mime = (type.split(";")[0] ?? "").trim().toLowerCase();
+  if (mime.startsWith("multipart/")) {
+    const boundary = param(type, "boundary");
+    if (boundary === undefined) return;
+    const delimiter = `--${boundary}`;
+    let current: string[] | undefined;
+    for (const line of body.split(/\r?\n/)) {
+      if (line.startsWith(delimiter)) {
+        if (current !== undefined) walk(current.join("\r\n"), plain, html, depth + 1);
+        // After the closing delimiter comes the epilogue, which is not content
+        if (line.startsWith(`${delimiter}--`)) return;
+        current = [];
+        continue;
+      }
+      current?.push(line);
+    }
+    if (current !== undefined) walk(current.join("\r\n"), plain, html, depth + 1);
+    return;
+  }
+  if (!mime.startsWith("text/")) return;
+  if (/^attachment/i.test(headers.get("content-disposition") ?? "")) return;
+  const decoded = decode(body, headers.get("content-transfer-encoding"), param(type, "charset"));
+  if (mime === "text/html") html.push(decoded);
+  else if (mime === "text/plain") plain.push(decoded);
+};
+
+const asText = (raw: Uint8Array | string): string => (typeof raw === "string" ? raw : Buffer.from(raw).toString("latin1"));
+
+/** This function gives the top-level headers of a raw message, unfolded, with lowercase names. */
+export const messageHeaders = (raw: Uint8Array | string): ReadonlyMap<string, string> => split(asText(raw)).headers;
+
+/** This function decodes the RFC 2047 encoded words (`=?UTF-8?Q?...?=`) of a header value. */
+export const decodeWords = (value: string): string =>
+  value.replace(/\?=\s+=\?/g, "?==?").replace(/=\?([^?]+)\?([BQ])\?([^?]*)\?=/gi, (_, charset: string, encoding: string, text: string) => {
+    const bytes =
+      encoding.toUpperCase() === "B" ? Buffer.from(text, "base64") : Buffer.from(quotedPrintable(text.replace(/_/g, " ")), "latin1");
+    return /^utf-?8$/i.test(charset) ? bytes.toString("utf8") : bytes.toString("latin1");
+  });
+
+/** This function gives the text of a message: its plain parts, or its HTML as lines when it has no plain part. */
+export const messageText = (raw: Uint8Array | string): string => {
+  const plain: string[] = [];
+  const html: string[] = [];
+  walk(asText(raw), plain, html, 0);
+  if (plain.some((part) => part.trim().length > 0)) {
+    return plain
+      .join("\n")
+      .replace(/\r\n/g, "\n")
+      .replace(/[ \t ]+/g, " ")
+      .trim();
+  }
+  return html.map(htmlText).join("\n").trim();
+};
+
+/** An address of a header: `Name <user@host>`, `<user@host>` or `user@host`. */
+export interface MailAddress {
+  readonly name?: string | undefined;
+  readonly address: string;
+}
+
+/** This function reads the addresses of an address header (From, To, Cc). */
+export const parseAddresses = (value: string): MailAddress[] => {
+  const out: MailAddress[] = [];
+  // Commas inside quotes do not separate addresses
+  for (const part of value.match(/(?:"[^"]*"|[^,])+/g) ?? []) {
+    const text = decodeWords(part.trim());
+    const angle = /^(.*?)<([^<>]+)>\s*$/.exec(text);
+    const address = (angle ? angle[2] : text)?.trim() ?? "";
+    if (!address.includes("@")) continue;
+    const name = angle?.[1]?.trim().replace(/^"|"$/g, "").trim();
+    out.push(name ? { name, address } : { address });
+  }
+  return out;
+};

package/src/module.ts

@@ -0,0 +1,18 @@
+/**
+ * The procedures of @procedures/mail as data. Importing this file registers nothing.
+ */
+
+import { defineModule, registerModules, PROCEDURE_REGISTRY, type ProcedureModule, type ProcedureRegistry } from "@procedures/client";
+import { mailProcedures } from "./procedures.js";
+
+export const mailModule: ProcedureModule = defineModule("@procedures/mail", mailProcedures);
+
+export default mailModule;
+
+/**
+ * This function registers the procedures of @procedures/mail in a registry. The default is the
+ * global registry.
+ */
+export function registerMailProcedures(registry: ProcedureRegistry = PROCEDURE_REGISTRY): void {
+  registerModules(registry, [mailModule]);
+}

package/src/procedures.ts

@@ -0,0 +1,139 @@
+/**
+ * The procedures of `@procedures/mail`: `mail.parse` and `mail.headers`.
+ */
+
+import { createProcedure, zodAdapter, type AnyProcedure, type ProcedureContext } from "@procedures/client";
+import { z } from "zod";
+import { readHeaders, type ImapClientFactory, type MailHeader } from "./imap.js";
+import { decodeWords, messageHeaders, messageText, parseAddresses, type MailAddress } from "./mime.js";
+
+const addressSchema = z.object({ name: z.string().optional(), address: z.string() });
+
+// =============================================================================
+// mail.parse
+// =============================================================================
+
+const ParseInputSchema = z.object({
+  raw: z.string().describe("The raw message (RFC 822): the headers, an empty line, the body"),
+});
+type ParseInput = z.infer<typeof ParseInputSchema>;
+
+interface ParseOutput {
+  headers: Record<string, string>;
+  subject?: string | undefined;
+  from: MailAddress[];
+  to: MailAddress[];
+  date?: string | undefined;
+  text: string;
+}
+const ParseOutputSchema = z.object({
+  headers: z.record(z.string(), z.string()),
+  subject: z.string().optional(),
+  from: z.array(addressSchema),
+  to: z.array(addressSchema),
+  date: z.string().optional(),
+  text: z.string(),
+});
+
+/** This function gives the headers, the addresses and the text of a raw message. */
+export function parseMessage(raw: string): ParseOutput {
+  const headers = messageHeaders(raw);
+  const subject = headers.get("subject");
+  const dateHeader = headers.get("date");
+  const date = dateHeader !== undefined && !Number.isNaN(Date.parse(dateHeader)) ? new Date(dateHeader).toISOString() : undefined;
+  return {
+    headers: Object.fromEntries(headers),
+    ...(subject !== undefined ? { subject: decodeWords(subject) } : {}),
+    from: parseAddresses(headers.get("from") ?? ""),
+    to: parseAddresses(headers.get("to") ?? ""),
+    ...(date !== undefined ? { date } : {}),
+    text: messageText(raw),
+  };
+}
+
+export const mailParseProcedure: AnyProcedure = createProcedure()
+  .path(["mail", "parse"])
+  .input(zodAdapter<ParseInput>(ParseInputSchema))
+  .output(zodAdapter<ParseOutput>(ParseOutputSchema))
+  .meta({
+    description: "Read the headers, the addresses and the text of a raw email message",
+    output: "json",
+    effects: { reads: [], writes: [] },
+  })
+  .handler(async (input: ParseInput): Promise<ParseOutput> => parseMessage(input.raw))
+  .build() as AnyProcedure;
+
+// =============================================================================
+// mail.headers
+// =============================================================================
+
+const HeadersInputSchema = z.object({
+  host: z.string().min(1).describe("The IMAP server, for example imap.gmail.com"),
+  port: z.number().int().min(1).max(65_535).default(993).describe("The port (default: 993)"),
+  secure: z.boolean().default(true).describe("Use TLS from the start (default: true)"),
+  user: z.string().min(1).describe("The user name of the mailbox"),
+  passwordEnv: z
+    .string()
+    .regex(/^[A-Za-z_][A-Za-z0-9_]*$/)
+    .default("IMAP_PASSWORD")
+    .describe("The name of the environment variable that holds the password. The password is never an input, so no log records it."),
+  mailbox: z.string().min(1).default("INBOX").describe("The mailbox (default: INBOX)"),
+  sinceDays: z.number().int().min(1).max(365).default(1).describe("The messages of this number of days (default: 1)"),
+  limit: z.number().int().min(1).max(5_000).default(200).describe("The maximum number of messages, the newest first (default: 200)"),
+  timeoutMs: z.number().int().positive().max(600_000).default(60_000).describe("The time limit, in milliseconds (default: 60000)"),
+});
+type HeadersInput = z.infer<typeof HeadersInputSchema>;
+
+interface HeadersOutput {
+  messages: MailHeader[];
+}
+const HeadersOutputSchema = z.object({
+  messages: z.array(
+    z.object({
+      uid: z.number(),
+      subject: z.string(),
+      from: z.array(addressSchema),
+      to: z.array(addressSchema),
+      date: z.string().optional(),
+      seen: z.boolean(),
+    })
+  ),
+});
+
+/** Tests replace the IMAP client. */
+let clientFactory: ImapClientFactory | undefined;
+
+/** This function replaces the IMAP client of `mail.headers` (for tests). Undefined restores `imapflow`. */
+export function setImapClientFactory(factory: ImapClientFactory | undefined): void {
+  clientFactory = factory;
+}
+
+export const mailHeadersProcedure: AnyProcedure = createProcedure()
+  .path(["mail", "headers"])
+  .input(zodAdapter<HeadersInput>(HeadersInputSchema))
+  .output(zodAdapter<HeadersOutput>(HeadersOutputSchema))
+  .meta({
+    description: "Give the headers of the recent messages of an IMAP mailbox, read only (no message becomes read)",
+    args: ["host", "user"],
+    output: "json",
+    effects: { reads: ["network"], writes: [] },
+  })
+  .handler(async (input: HeadersInput, ctx?: ProcedureContext): Promise<HeadersOutput> => {
+    const password = process.env[input.passwordEnv];
+    if (!password) throw new Error(`Set the environment variable ${input.passwordEnv} to the password of the mailbox`);
+    const messages = await readHeaders(
+      { host: input.host, port: input.port, secure: input.secure, user: input.user, password },
+      {
+        mailbox: input.mailbox,
+        since: new Date(Date.now() - input.sinceDays * 86_400_000),
+        limit: input.limit,
+        timeoutMs: input.timeoutMs,
+        signal: ctx?.signal,
+        clientFactory,
+      }
+    );
+    return { messages };
+  })
+  .build() as AnyProcedure;
+
+export const mailProcedures: AnyProcedure[] = [mailParseProcedure, mailHeadersProcedure];

package/src/register.ts

@@ -0,0 +1,10 @@
+/**
+ * Importing this file registers the procedures of @procedures/mail in the global registry:
+ * `import "@procedures/mail/register"`. The package root registers nothing.
+ */
+
+import { registerMailProcedures } from "./module.js";
+
+export * from "./module.js";
+
+registerMailProcedures();