agent-inbox 0.2.4 → 0.2.5

package/src/mail/fs-attachment-store.ts

@@ -1,163 +0,0 @@
-/**
- * Filesystem-backed AttachmentStore — opt-in for large/high-volume deployments.
- *
- * Keeps attachment bytes out of the SQLite DB (no WAL bloat, no synchronous
- * blob I/O blocking the event loop). Content-addressed by sha256, so identical
- * bytes dedup. Files live at `<dir>/<aa>/<sha256>` (sharded by the first byte
- * to avoid one enormous directory).
- *
- * Durability: writes go to a unique temp file, are fsync'd, then atomically
- * renamed into place. Because the caller stores the attachment (this `put`)
- * BEFORE committing the message row that references it, a crash can only ever
- * leave an orphan file (reclaimable by GC) — never a row pointing at missing
- * bytes. See docs/MAIL-INTEROP-PLAN.md §4c.
- */
-
-import { createHash } from "node:crypto";
-import * as fs from "node:fs/promises";
-import * as path from "node:path";
-import type { AttachmentStore } from "./types.js";
-
-const REF_RE = /^sha256:([0-9a-f]{64})$/;
-
-export class FsAttachmentStore implements AttachmentStore {
-  constructor(private dir: string) {}
-
-  async put(
-    bytes: Buffer,
-    _meta: { contentType: string; filename?: string }
-  ): Promise<string> {
-    const hex = createHash("sha256").update(bytes).digest("hex");
-    const ref = `sha256:${hex}`;
-    const full = this.pathForHex(hex);
-
-    // Content-addressed: if the bytes are already stored, we're done.
-    if (await exists(full)) return ref;
-
-    await fs.mkdir(path.dirname(full), { recursive: true });
-
-    // Atomic durable write: unique temp file → fsync → rename into place.
-    const tmp = `${full}.tmp-${process.pid}-${Date.now()}-${Math.random()
-      .toString(36)
-      .slice(2)}`;
-    let handle: fs.FileHandle | undefined;
-    try {
-      handle = await fs.open(tmp, "wx");
-      await handle.writeFile(bytes);
-      await handle.sync();
-    } finally {
-      await handle?.close();
-    }
-    try {
-      await fs.rename(tmp, full);
-    } catch (err) {
-      // A concurrent put of identical bytes may have won the race — that's fine
-      // (same content). Clean up our temp file and return the ref.
-      await fs.rm(tmp, { force: true });
-      if (!(await exists(full))) throw err;
-    }
-    // Durability: fsync the containing directory so the rename survives a crash
-    // (atomic rename alone is not crash-durable without a parent-dir fsync).
-    await fsyncDir(path.dirname(full));
-    return ref;
-  }
-
-  async get(ref: string): Promise<Buffer> {
-    const full = this.pathForRef(ref);
-    try {
-      return await fs.readFile(full);
-    } catch (err) {
-      if ((err as NodeJS.ErrnoException).code === "ENOENT") {
-        throw new Error(`Attachment not found: ${ref}`);
-      }
-      throw err;
-    }
-  }
-
-  async delete(ref: string): Promise<void> {
-    await fs.rm(this.pathForRef(ref), { force: true });
-  }
-
-  /**
-   * Garbage-collect orphan files: delete any stored attachment whose ref is not
-   * in `liveRefs` AND that is older than `minAgeMs` (default 1h).
-   *
-   * The age guard prevents a TOCTOU race: an attachment is `put` (file lands on
-   * disk) before the message row that references it is committed, so a just-
-   * written file may legitimately not appear in a `liveRefs` snapshot yet.
-   * Skipping recently-written files avoids deleting such in-flight attachments.
-   */
-  async gc(liveRefs: Set<string>, minAgeMs: number = 3_600_000): Promise<number> {
-    let removed = 0;
-    const cutoff = Date.now() - minAgeMs;
-    let shards: Array<{ name: string; isDirectory(): boolean }>;
-    try {
-      shards = await fs.readdir(this.dir, { withFileTypes: true });
-    } catch (err) {
-      if ((err as NodeJS.ErrnoException).code === "ENOENT") return 0;
-      throw err;
-    }
-    for (const shard of shards) {
-      if (!shard.isDirectory()) continue; // ignore stray files at the root
-      const shardDir = path.join(this.dir, shard.name);
-      let files: string[];
-      try {
-        files = await fs.readdir(shardDir);
-      } catch {
-        continue;
-      }
-      for (const name of files) {
-        if (name.includes(".tmp-")) continue; // leave in-flight temp files
-        if (liveRefs.has(`sha256:${name}`)) continue;
-        const filePath = path.join(shardDir, name);
-        try {
-          const st = await fs.stat(filePath);
-          if (st.mtimeMs > cutoff) continue; // too new — may be mid-ingestion
-        } catch {
-          continue;
-        }
-        await fs.rm(filePath, { force: true });
-        removed++;
-      }
-    }
-    return removed;
-  }
-
-  private pathForHex(hex: string): string {
-    // Shard by the first byte of the hash to bound directory fan-out.
-    return path.join(this.dir, hex.slice(0, 2), hex);
-  }
-
-  private pathForRef(ref: string): string {
-    const m = REF_RE.exec(ref);
-    if (!m) {
-      // Strict validation prevents path traversal via a crafted ref (refs are
-      // always sha256-of-content, never caller-chosen).
-      throw new Error(`Invalid attachment ref: ${ref}`);
-    }
-    return this.pathForHex(m[1]);
-  }
-}
-
-async function exists(p: string): Promise<boolean> {
-  try {
-    await fs.access(p);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-/** fsync a directory so a rename into it is durable. Best-effort across platforms. */
-async function fsyncDir(dir: string): Promise<void> {
-  let handle: fs.FileHandle | undefined;
-  try {
-    handle = await fs.open(dir, "r");
-    await handle.sync();
-  } catch {
-    // Some platforms (e.g. Windows) don't permit fsync on a directory handle.
-    // Atomic rename still holds; this only weakens the durability guarantee.
-  } finally {
-    await handle?.close().catch(() => {});
-  }
-}

package/src/mail/mail-gateway.ts

@@ -1,505 +0,0 @@
-/**
- * MailGateway — the only mail-aware component the rest of the system touches.
- *
- * Ingress: transport.onReceive → policy (size / allow-list / DMARC) → dedup →
- * bounce-detect → map → commit via router.routeMessage (single ingestion path).
- * The transport ACKs only after this resolves (commit-before-ACK); a throw
- * NACKs so the sender retries.
- *
- * Egress: router calls send() for mail-class recipients. The message is rendered
- * and attempted; transient failures are queued durably and retried; permanent
- * failures (and inbound DSNs) converge on one config-gated bounce handler.
- *
- * See docs/MAIL-INTEROP-PLAN.md §7, §8.
- */
-
-import { EventEmitter } from "node:events";
-import { createHash } from "node:crypto";
-import type { Storage } from "../storage/interface.js";
-import type { MessageRouter, MailEgress } from "../router/message-router.js";
-import { parseAddress } from "../federation/address.js";
-import { domainMatches } from "../router/destination.js";
-import { DeliveryQueue } from "../federation/delivery-queue.js";
-import type { QueueStore } from "../federation/queue-store.js";
-import { AddressBook } from "./address-book.js";
-import { RateLimiter } from "./rate-limiter.js";
-import {
-  inboundMailToMessage,
-  messageToOutboundMail,
-  UnknownRecipientError,
-} from "./email-mapper.js";
-import type { Message, MailIdentityConfig, DeliveryQueueConfig } from "../types.js";
-import type { MailTransport, InboundMail } from "./types.js";
-
-export interface MailGatewayConfig {
-  identity: MailIdentityConfig;
-  /** Domains we send to without erroring; combined with identity.localDomains. */
-  routableDomains?: string[];
-  /** Override the transport's maxMessageBytes for inbound rejection. */
-  maxMessageBytes?: number;
-  /** Inbound sender allow-list (domains). Empty/undefined = allow all. */
-  allowedSenderDomains?: string[];
-  /** Drop inbound mail whose DMARC verdict is "fail". */
-  rejectDmarcFail?: boolean;
-  /** Max attachments on an inbound message before it is rejected. */
-  maxAttachments?: number;
-  /**
-   * Inbound rate limiting (abuse control). Over-limit inbound is NACKed
-   * (transient) so legitimate senders back off and retry. Disabled if unset.
-   */
-  rateLimit?: {
-    /** Sliding window, ms (default 60000). */
-    windowMs?: number;
-    /** Max inbound per sender domain per window. */
-    perSenderDomain?: number;
-    /** Max inbound across all senders per window. */
-    global?: number;
-  };
-  /** Bounce behavior; both paths default on. */
-  bounce?: { emitEvent?: boolean; synthesizeInboxMessage?: boolean };
-  /** Domain used to mint outbound Message-IDs (defaults to primary local domain). */
-  messageIdDomain?: string;
-  /** Default inbox scope for inbound mail without plus-addressing. */
-  defaultScope?: string;
-  /** Delivery queue config for outbound retries. */
-  queue?: Partial<DeliveryQueueConfig>;
-}
-
-/** Single queue bucket for outbound mail retries. */
-const MAIL_QUEUE = "mail";
-
-/** Thrown to NACK over-limit inbound so the sender backs off and retries. */
-export class RateLimitedError extends Error {
-  constructor(public readonly scope: string) {
-    super(`Inbound rate limit exceeded (${scope})`);
-    this.name = "RateLimitedError";
-  }
-}
-
-export class MailGateway implements MailEgress {
-  private transport: MailTransport;
-  private storage: Storage;
-  private router: MessageRouter;
-  private events: EventEmitter;
-  private config: MailGatewayConfig;
-  private addressBook: AddressBook;
-  private mailDomains: string[];
-  private senderLimiter: RateLimiter | null = null;
-  private globalLimiter: RateLimiter | null = null;
-  readonly queue: DeliveryQueue;
-  private retryTimer?: ReturnType<typeof setInterval>;
-  private retrying = false;
-
-  constructor(opts: {
-    transport: MailTransport;
-    storage: Storage;
-    router: MessageRouter;
-    events: EventEmitter;
-    config: MailGatewayConfig;
-    /** Durable queue store for outbound retries (sqlite). Memory if omitted. */
-    queueStore?: QueueStore;
-  }) {
-    this.transport = opts.transport;
-    this.storage = opts.storage;
-    this.router = opts.router;
-    this.events = opts.events;
-    this.config = opts.config;
-    this.addressBook = new AddressBook(opts.config.identity);
-    if (this.addressBook.conflicts.length > 0) {
-      this.events.emit("mail.config.conflict", {
-        kind: "address-mapping",
-        conflicts: this.addressBook.conflicts,
-      });
-    }
-    this.mailDomains = [
-      ...opts.config.identity.localDomains,
-      ...(opts.config.routableDomains ?? []),
-    ];
-    this.queue = new DeliveryQueue(
-      opts.events,
-      opts.config.queue,
-      opts.queueStore ? { store: opts.queueStore } : undefined
-    );
-
-    const rl = opts.config.rateLimit;
-    if (rl) {
-      const windowMs = rl.windowMs ?? 60_000;
-      if (rl.perSenderDomain && rl.perSenderDomain > 0) {
-        this.senderLimiter = new RateLimiter(windowMs, rl.perSenderDomain);
-      }
-      if (rl.global && rl.global > 0) {
-        this.globalLimiter = new RateLimiter(windowMs, rl.global);
-      }
-    }
-
-    this.transport.onReceive((mail) => this.handleInbound(mail));
-  }
-
-  /** Start the underlying transport and the retry/TTL timers. */
-  async start(intervalMs: number = 30_000): Promise<void> {
-    await this.transport.start();
-    this.queue.startTicking();
-    this.retryTimer = setInterval(() => {
-      void this.processRetries();
-    }, intervalMs);
-  }
-
-  async stop(): Promise<void> {
-    if (this.retryTimer) clearInterval(this.retryTimer);
-    this.retryTimer = undefined;
-    this.queue.stopTicking();
-    await this.transport.stop();
-  }
-
-  // -- MailEgress ----------------------------------------------------------
-
-  /** True if this recipient's domain is one we route mail for. */
-  ownsRecipient(agentId: string): boolean {
-    if (agentId.indexOf("@") === -1) return false;
-    const addr = parseAddress(agentId);
-    return addr.system ? domainMatches(addr.system, this.mailDomains) : false;
-  }
-
-  /** Render and deliver the mail-class recipients of a message. */
-  async send(message: Message): Promise<void> {
-    const mailRecipients = message.recipients.filter((r) =>
-      this.ownsRecipient(r.agent_id)
-    );
-    if (mailRecipients.length === 0) return;
-    await this.attemptDelivery({ ...message, recipients: mailRecipients });
-  }
-
-  // -- Ingress -------------------------------------------------------------
-
-  /**
-   * Handle one inbound message. Resolves on ACK; throws on NACK (transient
-   * failure the sender should retry). Policy rejections ACK-and-drop so the
-   * sender does not retry forever.
-   */
-  async handleInbound(mail: InboundMail): Promise<void> {
-    const cap =
-      this.config.maxMessageBytes ?? this.transport.capabilities.maxMessageBytes;
-    if (mail.sizeBytes > cap) {
-      this.reject(mail, "size-exceeded");
-      return;
-    }
-
-    // Rate limiting (abuse control) — applies to all inbound including bounces.
-    // Over-limit NACKs (throws) so legitimate senders back off and retry.
-    this.enforceRateLimits(mail);
-
-    // Bounces/DSNs correlate to a prior send rather than routing as new mail.
-    if (mail.bounce) {
-      await this.handleInboundBounce(mail);
-      return;
-    }
-
-    if (!this.senderAllowed(mail)) {
-      this.reject(mail, "sender-not-allowed");
-      return;
-    }
-
-    if (this.config.rejectDmarcFail && mail.authResults?.dmarc === "fail") {
-      this.reject(mail, "dmarc-fail");
-      return;
-    }
-
-    const maxAtt = this.config.maxAttachments;
-    if (maxAtt !== undefined && (mail.attachments?.length ?? 0) > maxAtt) {
-      this.reject(mail, "too-many-attachments");
-      return;
-    }
-
-    const externalId = this.externalIdFor(mail);
-    if (this.storage.hasSeenExternalId(externalId)) {
-      this.events.emit("mail.duplicate", { externalId });
-      return; // already ingested — ACK without re-injecting
-    }
-
-    let mapped: Message;
-    try {
-      mapped = inboundMailToMessage(mail, {
-        addressBook: this.addressBook,
-        defaultScope: this.config.defaultScope,
-        lookupInboxIdByRfcMessageId: (id) =>
-          this.storage.getMessageIdByExternalId(id),
-      });
-    } catch (err) {
-      if (err instanceof UnknownRecipientError) {
-        this.reject(mail, "no-local-recipient");
-        return;
-      }
-      throw err; // unexpected — NACK
-    }
-
-    // Single ingestion path: routeMessage stores, marks delivery, fires events.
-    // When the reply resolves to a known parent, inherit the parent's thread_tag
-    // (keeps threads consistent even for clients that send In-Reply-To without a
-    // full References chain). Fall back to our derived tag if the parent is gone.
-    const parentThreadTag = mapped.in_reply_to
-      ? this.storage.getMessage(mapped.in_reply_to)?.thread_tag
-      : undefined;
-    const stored = await this.router.routeMessage({
-      from: mapped.sender_id,
-      to: mapped.recipients.map((r) => ({ agent_id: r.agent_id, kind: r.kind })),
-      payload: mapped.content,
-      scope: mapped.scope,
-      subject: mapped.subject,
-      threadTag: parentThreadTag ?? mapped.thread_tag,
-      inReplyTo: mapped.in_reply_to,
-      importance: mapped.importance,
-      metadata: mapped.metadata,
-    });
-
-    this.storage.recordExternalId(externalId, stored.id);
-    this.events.emit("mail.received", { messageId: stored.id, externalId });
-  }
-
-  // -- Egress delivery -----------------------------------------------------
-
-  /** Attempt delivery; classify the result and queue/bounce accordingly. */
-  private async attemptDelivery(
-    message: Message,
-    queuedId?: string
-  ): Promise<void> {
-    const outbound = messageToOutboundMail(message, {
-      addressBook: this.addressBook,
-      messageIdDomain: this.config.messageIdDomain,
-      lookupRfcMessageId: (id) =>
-        this.storage.getMessage(id)?.metadata.rfcMessageId as
-          | string
-          | undefined,
-      lookupReferences: (id) =>
-        this.storage.getMessage(id)?.metadata.rfcReferences as
-          | string[]
-          | undefined,
-    });
-
-    let disposition: "delivered" | "transient" | "permanent";
-    let detail: string | undefined;
-    let code: number | undefined;
-    let remoteMessageId: string | undefined;
-    try {
-      const res = await this.transport.send(outbound);
-      disposition = res.disposition;
-      detail = res.detail;
-      code = res.code;
-      remoteMessageId = res.remoteMessageId;
-    } catch (err) {
-      // Transport threw (e.g. not ready) — treat as transient, keep retrying.
-      disposition = "transient";
-      detail = err instanceof Error ? err.message : String(err);
-    }
-
-    if (disposition === "delivered") {
-      // Record our Message-ID so an inbound DSN or reply can correlate back.
-      this.storage.recordExternalId(outbound.headers.messageId, message.id);
-      // Also record the provider's id — some backends (Postmark) reference their
-      // own message id in bounce webhooks rather than the RFC Message-ID.
-      if (remoteMessageId) {
-        this.storage.recordExternalId(remoteMessageId, message.id);
-      }
-      this.markDelivered(message);
-      if (queuedId) this.queue.removeEntry(MAIL_QUEUE, queuedId);
-      this.events.emit("mail.sent", { messageId: message.id, remoteMessageId });
-      return;
-    }
-
-    if (disposition === "transient") {
-      if (queuedId) {
-        // recordAttempt returns false once retryMaxAttempts is exhausted (the
-        // entry is then removed). Bounce instead of silently dropping it.
-        const kept = this.queue.recordAttempt(MAIL_QUEUE, queuedId);
-        if (!kept) {
-          await this.bounce({
-            message,
-            recipient: outbound.to.map((t) => t.address).join(", "),
-            status: code ? String(code) : undefined,
-            diagnostic: detail ?? "retries exhausted",
-            action: "failed",
-          });
-          return;
-        }
-      } else {
-        this.queue.enqueue(MAIL_QUEUE, message);
-      }
-      this.events.emit("mail.queued", { messageId: message.id, detail });
-      return;
-    }
-
-    // permanent
-    if (queuedId) this.queue.removeEntry(MAIL_QUEUE, queuedId);
-    await this.bounce({
-      message,
-      recipient: outbound.to.map((t) => t.address).join(", "),
-      status: code ? String(code) : undefined,
-      diagnostic: detail,
-      action: "failed",
-    });
-  }
-
-  /** Re-attempt all retryable queued mail. Call from a timer or deterministically. */
-  async processRetries(): Promise<void> {
-    // Guard against overlapping ticks: a slow send must not be re-selected and
-    // re-sent by the next interval while it is still in flight.
-    if (this.retrying) return;
-    this.retrying = true;
-    // Opportunistically reclaim idle rate-limiter keys (bounded memory).
-    this.senderLimiter?.prune();
-    this.globalLimiter?.prune();
-    try {
-      for (const entry of this.queue.getRetryable(MAIL_QUEUE)) {
-        await this.attemptDelivery(entry.message, entry.id);
-      }
-    } finally {
-      this.retrying = false;
-    }
-  }
-
-  private markDelivered(message: Message): void {
-    const stored = this.storage.getMessage(message.id);
-    if (!stored) return;
-    const now = new Date().toISOString();
-    for (const r of stored.recipients) {
-      if (this.ownsRecipient(r.agent_id)) r.delivered_at = now;
-    }
-    this.storage.putMessage(stored);
-  }
-
-  // -- Bounce (shared by outbound permanent + inbound DSN) -----------------
-
-  private async handleInboundBounce(mail: InboundMail): Promise<void> {
-    const b = mail.bounce!;
-    await this.bounce({
-      originalMessageId: b.originalMessageId,
-      recipient: b.recipient,
-      status: b.status,
-      diagnostic: b.diagnostic,
-      action: b.action,
-    });
-  }
-
-  private async bounce(opts: {
-    message?: Message;
-    originalMessageId?: string;
-    recipient: string;
-    status?: string;
-    diagnostic?: string;
-    action: "failed" | "delayed" | "delivered";
-  }): Promise<void> {
-    let original = opts.message;
-    if (!original && opts.originalMessageId) {
-      const inboxId = this.storage.getMessageIdByExternalId(
-        opts.originalMessageId
-      );
-      if (inboxId) original = this.storage.getMessage(inboxId);
-    }
-
-    // A delayed DSN means the peer MTA is still retrying — surface, don't bounce.
-    if (opts.action === "delayed") {
-      this.events.emit("mail.queued", {
-        messageId: original?.id,
-        recipient: opts.recipient,
-        delayed: true,
-      });
-      return;
-    }
-    // A positive DSN ("delivered") is informational only.
-    if (opts.action === "delivered") return;
-
-    // Dedup: a replayed/spoofed DSN echoing a known message id must not
-    // repeatedly emit events or synthesize inbox notices (flood protection).
-    const bounceKey = `bounce:${original?.id ?? opts.originalMessageId ?? "?"}:${opts.recipient}`;
-    if (this.storage.hasSeenExternalId(bounceKey)) return;
-    this.storage.recordExternalId(bounceKey, original?.id ?? bounceKey);
-
-    if (this.config.bounce?.emitEvent !== false) {
-      this.events.emit("mail.bounced", {
-        messageId: original?.id,
-        recipient: opts.recipient,
-        status: opts.status,
-        diagnostic: opts.diagnostic,
-      });
-    }
-
-    if (this.config.bounce?.synthesizeInboxMessage !== false && original) {
-      await this.synthesizeBounce(original, opts);
-    }
-  }
-
-  /** Inject a bounce notification into the original sender's inbox. */
-  private async synthesizeBounce(
-    original: Message,
-    opts: { recipient: string; status?: string; diagnostic?: string }
-  ): Promise<void> {
-    const parts = [`Delivery to ${opts.recipient} failed`];
-    if (opts.status) parts.push(`(${opts.status})`);
-    if (opts.diagnostic) parts.push(`- ${opts.diagnostic}`);
-
-    await this.router.routeMessage({
-      from: "mailer-daemon",
-      to: [{ agent_id: original.sender_id, kind: "to" }],
-      subject: `Delivery failed: ${original.subject ?? "(no subject)"}`,
-      payload: { type: "text", text: parts.join(" ") },
-      inReplyTo: original.id,
-      importance: "high",
-      scope: original.scope,
-      metadata: {
-        bounce: {
-          recipient: opts.recipient,
-          status: opts.status,
-          diagnostic: opts.diagnostic,
-        },
-      },
-    });
-  }
-
-  // -- Helpers -------------------------------------------------------------
-
-  private reject(mail: InboundMail, reason: string): void {
-    this.events.emit("mail.rejected", {
-      reason,
-      from: mail.from.address,
-      to: mail.to.map((t) => t.address),
-    });
-  }
-
-  /** Throw RateLimitedError (→ NACK) when an inbound limiter is exceeded. */
-  private enforceRateLimits(mail: InboundMail): void {
-    if (this.globalLimiter && !this.globalLimiter.tryAcquire("*")) {
-      this.events.emit("mail.rate_limited", { scope: "global" });
-      throw new RateLimitedError("global");
-    }
-    if (this.senderLimiter) {
-      const from = mail.from.address || mail.envelopeFrom || "unknown";
-      const at = from.lastIndexOf("@");
-      const domain = at === -1 ? from : from.slice(at + 1).toLowerCase();
-      if (!this.senderLimiter.tryAcquire(domain)) {
-        this.events.emit("mail.rate_limited", { scope: "sender", domain });
-        throw new RateLimitedError(`sender:${domain}`);
-      }
-    }
-  }
-
-  private senderAllowed(mail: InboundMail): boolean {
-    const allow = this.config.allowedSenderDomains;
-    if (!allow || allow.length === 0) return true;
-    const at = mail.from.address.lastIndexOf("@");
-    if (at === -1) return false;
-    return domainMatches(mail.from.address.slice(at + 1), allow);
-  }
-
-  /** Dedup key: RFC Message-ID when present, else a synthetic content hash. */
-  private externalIdFor(mail: InboundMail): string {
-    if (mail.headers.messageId) return mail.headers.messageId;
-    const h = createHash("sha256");
-    h.update(mail.from.address);
-    h.update("\0");
-    h.update(mail.subject ?? "");
-    h.update("\0");
-    h.update(mail.receivedAt);
-    h.update("\0");
-    h.update((mail.text ?? mail.html ?? "").slice(0, 4096));
-    return "syn:" + h.digest("hex");
-  }
-}