agent-inbox 0.2.2 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/AGENTS.md +18 -0
  2. package/CLAUDE.md +92 -1
  3. package/README.md +73 -6
  4. package/bench/inbox-growth.bench.ts +224 -0
  5. package/dist/federation/connection-manager.d.ts +8 -0
  6. package/dist/federation/connection-manager.d.ts.map +1 -1
  7. package/dist/federation/connection-manager.js +12 -0
  8. package/dist/federation/connection-manager.js.map +1 -1
  9. package/dist/federation/delivery-queue.d.ts +11 -3
  10. package/dist/federation/delivery-queue.d.ts.map +1 -1
  11. package/dist/federation/delivery-queue.js +38 -8
  12. package/dist/federation/delivery-queue.js.map +1 -1
  13. package/dist/federation/queue-store.d.ts +42 -0
  14. package/dist/federation/queue-store.d.ts.map +1 -0
  15. package/dist/federation/queue-store.js +87 -0
  16. package/dist/federation/queue-store.js.map +1 -0
  17. package/dist/index.d.mts +2 -0
  18. package/dist/index.d.ts +29 -0
  19. package/dist/index.d.ts.map +1 -1
  20. package/dist/index.js +124 -1
  21. package/dist/index.js.map +1 -1
  22. package/dist/index.mjs +1 -0
  23. package/dist/index.mjs.map +1 -0
  24. package/dist/jsonrpc/mail-push-types.d.ts +9 -0
  25. package/dist/jsonrpc/mail-push-types.d.ts.map +1 -1
  26. package/dist/jsonrpc/mail-push-types.js +1 -0
  27. package/dist/jsonrpc/mail-push-types.js.map +1 -1
  28. package/dist/jsonrpc/mail-server.d.ts +8 -1
  29. package/dist/jsonrpc/mail-server.d.ts.map +1 -1
  30. package/dist/jsonrpc/mail-server.js +42 -1
  31. package/dist/jsonrpc/mail-server.js.map +1 -1
  32. package/dist/mail/address-book.d.ts +43 -0
  33. package/dist/mail/address-book.d.ts.map +1 -0
  34. package/dist/mail/address-book.js +95 -0
  35. package/dist/mail/address-book.js.map +1 -0
  36. package/dist/mail/attachment-store.d.ts +31 -0
  37. package/dist/mail/attachment-store.d.ts.map +1 -0
  38. package/dist/mail/attachment-store.js +74 -0
  39. package/dist/mail/attachment-store.js.map +1 -0
  40. package/dist/mail/email-mapper.d.ts +41 -0
  41. package/dist/mail/email-mapper.d.ts.map +1 -0
  42. package/dist/mail/email-mapper.js +216 -0
  43. package/dist/mail/email-mapper.js.map +1 -0
  44. package/dist/mail/fs-attachment-store.d.ts +38 -0
  45. package/dist/mail/fs-attachment-store.d.ts.map +1 -0
  46. package/dist/mail/fs-attachment-store.js +165 -0
  47. package/dist/mail/fs-attachment-store.js.map +1 -0
  48. package/dist/mail/mail-gateway.d.ts +114 -0
  49. package/dist/mail/mail-gateway.d.ts.map +1 -0
  50. package/dist/mail/mail-gateway.js +402 -0
  51. package/dist/mail/mail-gateway.js.map +1 -0
  52. package/dist/mail/provider-transport.d.ts +138 -0
  53. package/dist/mail/provider-transport.d.ts.map +1 -0
  54. package/dist/mail/provider-transport.js +434 -0
  55. package/dist/mail/provider-transport.js.map +1 -0
  56. package/dist/mail/rate-limiter.d.ts +20 -0
  57. package/dist/mail/rate-limiter.d.ts.map +1 -0
  58. package/dist/mail/rate-limiter.js +56 -0
  59. package/dist/mail/rate-limiter.js.map +1 -0
  60. package/dist/mail/smtp-transport.d.ts +141 -0
  61. package/dist/mail/smtp-transport.d.ts.map +1 -0
  62. package/dist/mail/smtp-transport.js +415 -0
  63. package/dist/mail/smtp-transport.js.map +1 -0
  64. package/dist/mail/types.d.ts +177 -0
  65. package/dist/mail/types.d.ts.map +1 -0
  66. package/dist/mail/types.js +11 -0
  67. package/dist/mail/types.js.map +1 -0
  68. package/dist/push/notifier.d.ts +21 -0
  69. package/dist/push/notifier.d.ts.map +1 -1
  70. package/dist/push/notifier.js +84 -2
  71. package/dist/push/notifier.js.map +1 -1
  72. package/dist/router/destination.d.ts +69 -0
  73. package/dist/router/destination.d.ts.map +1 -0
  74. package/dist/router/destination.js +106 -0
  75. package/dist/router/destination.js.map +1 -0
  76. package/dist/router/message-router.d.ts +15 -0
  77. package/dist/router/message-router.d.ts.map +1 -1
  78. package/dist/router/message-router.js +25 -3
  79. package/dist/router/message-router.js.map +1 -1
  80. package/dist/storage/interface.d.ts +21 -0
  81. package/dist/storage/interface.d.ts.map +1 -1
  82. package/dist/storage/memory.d.ts +12 -0
  83. package/dist/storage/memory.d.ts.map +1 -1
  84. package/dist/storage/memory.js +50 -0
  85. package/dist/storage/memory.js.map +1 -1
  86. package/dist/storage/sqlite.d.ts +14 -0
  87. package/dist/storage/sqlite.d.ts.map +1 -1
  88. package/dist/storage/sqlite.js +79 -1
  89. package/dist/storage/sqlite.js.map +1 -1
  90. package/dist/traceability/traceability.d.ts.map +1 -1
  91. package/dist/traceability/traceability.js +7 -17
  92. package/dist/traceability/traceability.js.map +1 -1
  93. package/dist/types.d.ts +80 -0
  94. package/dist/types.d.ts.map +1 -1
  95. package/docs/DESIGN.md +15 -0
  96. package/docs/MAIL-INTEROP-PLAN.md +660 -0
  97. package/package.json +29 -3
  98. package/renovate.json5 +6 -0
  99. package/rules/agent-inbox.md +1 -0
  100. package/src/federation/connection-manager.ts +12 -0
  101. package/src/federation/delivery-queue.ts +38 -8
  102. package/src/federation/queue-store.ts +124 -0
  103. package/src/index.ts +186 -1
  104. package/src/jsonrpc/mail-push-types.ts +10 -0
  105. package/src/jsonrpc/mail-server.ts +48 -1
  106. package/src/mail/address-book.ts +111 -0
  107. package/src/mail/attachment-store.ts +90 -0
  108. package/src/mail/email-mapper.ts +288 -0
  109. package/src/mail/fs-attachment-store.ts +163 -0
  110. package/src/mail/mail-gateway.ts +505 -0
  111. package/src/mail/provider-transport.ts +577 -0
  112. package/src/mail/rate-limiter.ts +51 -0
  113. package/src/mail/smtp-transport.ts +589 -0
  114. package/src/mail/types.ts +221 -0
  115. package/src/push/notifier.ts +98 -2
  116. package/src/router/destination.ts +140 -0
  117. package/src/router/message-router.ts +41 -4
  118. package/src/storage/interface.ts +22 -0
  119. package/src/storage/memory.ts +59 -0
  120. package/src/storage/sqlite.ts +114 -1
  121. package/src/traceability/traceability.ts +7 -16
  122. package/src/types.ts +74 -0
  123. package/test/federation/delivery-queue-sqlite.test.ts +158 -0
  124. package/test/load.test.ts +288 -0
  125. package/test/mail/address-book.test.ts +111 -0
  126. package/test/mail/attachment-store-contract.test.ts +92 -0
  127. package/test/mail/attachment-store.test.ts +69 -0
  128. package/test/mail/destination.test.ts +115 -0
  129. package/test/mail/dsn-parse.test.ts +239 -0
  130. package/test/mail/email-mapper.test.ts +341 -0
  131. package/test/mail/external-id.test.ts +43 -0
  132. package/test/mail/fs-attachment-store.test.ts +134 -0
  133. package/test/mail/full-flow-e2e.test.ts +200 -0
  134. package/test/mail/mail-gateway.test.ts +419 -0
  135. package/test/mail/mail-transport-contract.test.ts +134 -0
  136. package/test/mail/mock-mail.ts +161 -0
  137. package/test/mail/mock-postmark.ts +66 -0
  138. package/test/mail/provider-transport.test.ts +381 -0
  139. package/test/mail/rate-limiter.test.ts +48 -0
  140. package/test/mail/router-mail-integration.test.ts +138 -0
  141. package/test/mail/smtp-e2e.test.ts +98 -0
  142. package/test/mail/smtp-transport.test.ts +138 -0
  143. package/test/mail-presence.test.ts +149 -0
  144. package/test/mail-push.test.ts +44 -0
  145. package/test/mail-server.test.ts +25 -0
  146. package/test/push-notifier.test.ts +81 -0
  147. package/test/sqlite-storage.test.ts +106 -0
  148. package/test/storage.test.ts +92 -0
  149. package/vitest.bench.config.ts +8 -0
@@ -0,0 +1,660 @@
1
+ # Mail Interop — Implementation Plan
2
+
3
+ Status: **proposed** · Branch: `claude/smtp-durable-inbox-gtUjJ`
4
+
5
+ Bidirectional email interop for Agent Inbox: agents can receive email from and
6
+ send email to external addresses, through a **pluggable `MailTransport`
7
+ interface** with a **lightweight self-hosted SMTP backend as the default** and a
8
+ **managed-provider backend** (SES / Postmark / Mailgun) as a drop-in alternative.
9
+
10
+ This document focuses on **hardening the interface** — the contract every
11
+ backend must satisfy — because the interface is the thing we can't cheaply
12
+ change later. The two concrete backends are described only insofar as they
13
+ prove the contract holds for both a raw-SMTP world and a provider-API world.
14
+
15
+ ---
16
+
17
+ ## 0. Design goals for the interface
18
+
19
+ 1. **Backend-agnostic.** The router, gateway, and storage layer must never know
20
+ whether mail is moving over self-hosted SMTP or a provider API. Same pattern
21
+ as `MapConnection` / `MapAgentConnectionClass` letting mesh and WebSocket
22
+ coexist per-peer.
23
+ 2. **Durable by contract, not by convention.** Inbound is committed before the
24
+ protocol ACK; outbound is enqueued durably before the first send attempt.
25
+ The interface makes the durability handoff explicit so a backend cannot
26
+ "forget" to be safe.
27
+ 3. **Honest error taxonomy.** Backends classify every failure as `permanent` /
28
+ `transient` / `unknown`. The retry/queue layer is driven by that
29
+ classification, never by string-matching SMTP text.
30
+ 4. **Capability negotiation, not assumption.** Backends advertise what they can
31
+ do (DKIM signing, inbound auth verification, MX delivery, webhooks). The
32
+ gateway adapts instead of assuming.
33
+ 5. **Untrusted-input posture baked in.** Inbound mail is hostile by default:
34
+ size caps, allow-lists, and auth-result surfacing are interface-level, not
35
+ per-backend afterthoughts.
36
+
37
+ ---
38
+
39
+ ## 1. The address-collision problem (must fix first)
40
+
41
+ Email addresses are syntactically identical to federation addresses:
42
+
43
+ ```ts
44
+ parseAddress("bob@gmail.com") // → { agent: "bob", system: "gmail.com" }
45
+ isRemoteAddress(...) // → true ← would route into ConnectionManager
46
+ ```
47
+
48
+ Today `MessageRouter.routeMessage` (src/router/message-router.ts:94-105) sends
49
+ anything with a `system` to federation. An email recipient would be silently
50
+ swallowed by `ConnectionManager.route()` and dropped as "no route".
51
+
52
+ ### Resolution: a classified destination, computed once
53
+
54
+ Introduce an explicit recipient classification step that runs **before** the
55
+ federation branch. A destination is one of:
56
+
57
+ | Class | Shape | Routed to |
58
+ |--------------|--------------------------------|----------------------|
59
+ | `local` | `bob` (known agent) | storage inbox |
60
+ | `federation` | `bob@system-2` (known peer) | `ConnectionManager` |
61
+ | `mail` | `bob@example.com` (mail domain)| `MailGateway` |
62
+
63
+ **Decision (resolved): explicit-list classification, with federation-domain
64
+ support.** Nothing is guessed from "looks like a domain". A destination is mail
65
+ only if its domain is in an explicit list, federation only if its system is a
66
+ configured peer, and an unknown `@x` is a hard error rather than a silent guess.
67
+ Because some federation systems are legitimately addressed with domain-like ids
68
+ (e.g. `team.corp.internal`), the classifier supports a federation peer
69
+ *declaring its own domain(s)* so the same dotted form routes to federation when
70
+ that's what the operator meant.
71
+
72
+ Disambiguation rule (deterministic, in priority order):
73
+
74
+ 1. No `@` → `local`.
75
+ 2. `@<x>` where `<x>` matches a **configured federation peer `systemId`** *or*
76
+ one of that peer's declared `domains` → `federation`.
77
+ 3. `@<x>` where `<x>` (or its parent domain) is in the **configured mail
78
+ `localDomains` / `routableDomains`** → `mail`.
79
+ 4. Otherwise → **error** (`UnknownDestinationError`): the recipient is rejected
80
+ at routing time, surfaced to the sender, never silently dropped or guessed.
81
+
82
+ So a single dotted form like `@corp.internal` is `federation` if a peer claims
83
+ it, `mail` if mail config claims it, and an error if neither does. When a domain
84
+ appears in *both* (operator misconfiguration), **federation wins** and we emit a
85
+ `mail.config.conflict` warning at startup so it's visible.
86
+
87
+ This lives in a new `classifyDestination(agentId, ctx)` helper in
88
+ `src/router/destination.ts`, taking the set of known peer system ids + their
89
+ declared domains + mail config, so the rule is data-driven and testable in
90
+ isolation. **`parseAddress` is left untouched** — we add a layer above it rather
91
+ than overloading its meaning.
92
+
93
+ ```ts
94
+ export type Destination =
95
+ | { class: "local"; agentId: string }
96
+ | { class: "federation"; agentId: string; address: FederatedAddress }
97
+ | { class: "mail"; email: string; localAgentId?: string };
98
+
99
+ export interface DestinationContext {
100
+ isLocalAgent: (id: string) => boolean;
101
+ federationPeers: Array<{ systemId: string; domains?: string[] }>;
102
+ mailDomains: string[]; // localDomains ∪ routableDomains
103
+ }
104
+
105
+ /** Throws UnknownDestinationError for rule-4 misses. */
106
+ export function classifyDestination(agentId: string, ctx: DestinationContext): Destination;
107
+ ```
108
+
109
+ > Hardening note: the federation check (rule 2) runs before the mail check
110
+ > (rule 3) so a peer's declared domain always wins; the "no implicit guess"
111
+ > rule-4 error and the both-lists conflict warning are covered by tests.
112
+
113
+ ---
114
+
115
+ ## 2. The `MailTransport` interface
116
+
117
+ `src/mail/mail-transport.ts`
118
+
119
+ ```ts
120
+ /** What a backend can do. The gateway adapts to these. */
121
+ export interface MailCapabilities {
122
+ /** Backend can deliver to arbitrary public MX (self-hosted/SMTP) vs only
123
+ * via its own API/relay (some providers). Informational for routing. */
124
+ outbound: "mx" | "relay" | "api";
125
+ /** Backend signs DKIM itself (provider) or we must sign before handing off. */
126
+ signsDkim: boolean;
127
+ /** Backend verifies SPF/DKIM/DMARC on inbound and reports results. */
128
+ verifiesInboundAuth: boolean;
129
+ /** Inbound arrives via a long-lived listener (SMTP) or push webhook. */
130
+ inbound: "listener" | "webhook" | "none";
131
+ /** Max message size the backend will accept, bytes. */
132
+ maxMessageBytes: number;
133
+ }
134
+
135
+ export type MailTransportState = "stopped" | "starting" | "ready" | "degraded" | "stopping";
136
+
137
+ export interface MailTransport {
138
+ readonly capabilities: MailCapabilities;
139
+ readonly state: MailTransportState;
140
+
141
+ /** Idempotent. Resolves only when inbound is actually accepting (port bound /
142
+ * webhook registered) so callers don't ACK before we can receive. */
143
+ start(): Promise<void>;
144
+
145
+ /** Graceful: stop accepting new inbound, finish in-flight, then release. */
146
+ stop(): Promise<void>;
147
+
148
+ /**
149
+ * Send one already-rendered outbound message.
150
+ * MUST be idempotent on `envelope.idempotencyKey`: a retry with the same key
151
+ * must not produce a duplicate external email.
152
+ * MUST NOT throw for delivery failures — return a classified result instead.
153
+ * MAY throw only for programmer error (e.g. called while stopped).
154
+ */
155
+ send(envelope: OutboundMail): Promise<MailSendResult>;
156
+
157
+ /**
158
+ * Register the single ingress handler. The transport calls it once per
159
+ * received message and MUST await it. Resolution of the handler = "durably
160
+ * accepted"; the transport only then ACKs the peer MTA / webhook (250 / 200).
161
+ * A thrown handler = NACK (4xx) so the sender retries — never silent drop.
162
+ */
163
+ onReceive(handler: InboundHandler): void;
164
+
165
+ /** Liveness for health endpoints and the degraded → ready transition. */
166
+ health(): Promise<MailHealth>;
167
+ }
168
+
169
+ export type InboundHandler = (mail: InboundMail) => Promise<void>;
170
+ ```
171
+
172
+ ### Why these shapes harden the contract
173
+
174
+ - **`send` never throws on delivery failure.** Throw-vs-return is the most
175
+ common way pluggable transports leak backend quirks. Delivery outcomes are
176
+ *data* (`MailSendResult`), so the queue logic is identical for SMTP 4xx and a
177
+ provider's JSON `{ "error": "throttled" }`.
178
+ - **`onReceive` handler resolution gates the ACK.** This is the durability
179
+ contract in the type system: a backend physically cannot ACK before our
180
+ storage commit returns, because the ACK is downstream of the awaited handler.
181
+ - **`start()` resolves only when truly accepting.** Prevents the race where we
182
+ tell an agent "mail is on" before the socket is bound.
183
+ - **Capabilities are read, not assumed.** The gateway uses `signsDkim` to decide
184
+ whether to sign, `maxMessageBytes` to reject early, `verifiesInboundAuth` to
185
+ decide whether to trust the auth verdict or compute its own.
186
+
187
+ ---
188
+
189
+ ## 3. Envelope types — the wire contract
190
+
191
+ ```ts
192
+ export interface MailAddress {
193
+ /** RFC 5322 addr-spec, e.g. "bob@example.com". */
194
+ address: string;
195
+ /** Optional display name. */
196
+ name?: string;
197
+ }
198
+
199
+ export interface OutboundMail {
200
+ /** Stable key for idempotent send. Derived from inbox Message.id. */
201
+ idempotencyKey: string;
202
+ from: MailAddress;
203
+ to: MailAddress[];
204
+ cc?: MailAddress[];
205
+ bcc?: MailAddress[];
206
+ subject?: string;
207
+ /** Rendered bodies. At least one of text/html. */
208
+ text?: string;
209
+ html?: string;
210
+ /** RFC 5322 threading headers, already computed by the mapper. */
211
+ headers: {
212
+ messageId: string; // our Message-ID for this email
213
+ inReplyTo?: string; // parent's Message-ID
214
+ references?: string[]; // full chain
215
+ };
216
+ attachments?: MailAttachment[];
217
+ /** Opaque passthrough for backend-specific knobs (provider tags, etc.). */
218
+ backendHints?: Record<string, unknown>;
219
+ }
220
+
221
+ export interface InboundMail {
222
+ /** The peer-asserted envelope (MAIL FROM / RCPT TO), distinct from headers. */
223
+ envelopeFrom: string;
224
+ envelopeTo: string[];
225
+ from: MailAddress;
226
+ to: MailAddress[];
227
+ cc?: MailAddress[];
228
+ subject?: string;
229
+ text?: string;
230
+ html?: string;
231
+ headers: {
232
+ messageId?: string;
233
+ inReplyTo?: string;
234
+ references?: string[];
235
+ /** Everything else, lower-cased keys, for the mapper to mine. */
236
+ raw: Record<string, string | string[]>;
237
+ };
238
+ attachments?: MailAttachment[];
239
+ /** Auth verdict — present iff capabilities.verifiesInboundAuth. */
240
+ authResults?: InboundAuthResults;
241
+ /** Peer IP / hostname, for allow-listing and abuse control. */
242
+ remote?: { ip?: string; reverseDns?: string };
243
+ /**
244
+ * Bounce/DSN classification, parsed from `Content-Type: multipart/report;
245
+ * report-type=delivery-status` (RFC 3464). Present when this inbound mail is
246
+ * a bounce, so the gateway can correlate it to the failed outbound send and
247
+ * drive the §8 bounce path instead of treating it as a fresh message.
248
+ */
249
+ bounce?: InboundBounce;
250
+ sizeBytes: number;
251
+ receivedAt: string; // ISO
252
+ }
253
+
254
+ export interface InboundBounce {
255
+ /** "failed" (permanent), "delayed" (transient), "delivered" (positive DSN). */
256
+ action: "failed" | "delayed" | "delivered";
257
+ /** The address that bounced. */
258
+ recipient: string;
259
+ /** RFC 3463 status code, e.g. "5.1.1". */
260
+ status?: string;
261
+ /** Message-ID of the original outbound mail, when the DSN echoes it. */
262
+ originalMessageId?: string;
263
+ diagnostic?: string;
264
+ }
265
+
266
+ export interface InboundAuthResults {
267
+ spf: "pass" | "fail" | "softfail" | "neutral" | "none";
268
+ dkim: "pass" | "fail" | "none";
269
+ dmarc: "pass" | "fail" | "none";
270
+ }
271
+
272
+ export interface MailAttachment {
273
+ filename?: string;
274
+ contentType: string;
275
+ contentId?: string; // for inline html
276
+ /** Stored out-of-band; the envelope carries a handle, not megabytes. */
277
+ contentRef: string;
278
+ sizeBytes: number;
279
+ }
280
+ ```
281
+
282
+ ### `MailSendResult` — the error taxonomy that drives retries
283
+
284
+ ```ts
285
+ export type MailSendDisposition = "delivered" | "transient" | "permanent";
286
+
287
+ export interface MailSendResult {
288
+ disposition: MailSendDisposition;
289
+ /** Provider/SMTP id when delivered, for correlating DSNs/bounces. */
290
+ remoteMessageId?: string;
291
+ /** Numeric SMTP-style code when available (e.g. 250, 451, 550). */
292
+ code?: number;
293
+ /** Human-readable, for logs only — never parsed for control flow. */
294
+ detail?: string;
295
+ /** For transient: hint for next attempt (e.g. greylist retry-after). */
296
+ retryAfterMs?: number;
297
+ }
298
+ ```
299
+
300
+ - `delivered` → dequeue, mark recipient `delivered_at`.
301
+ - `transient` → keep in queue, schedule retry (honoring `retryAfterMs`), respect
302
+ `maxTTL` / `retryMaxAttempts` from `DeliveryQueueConfig`.
303
+ - `permanent` → dequeue, emit `mail.bounced`, optionally synthesize a bounce
304
+ message back into the sender's inbox.
305
+
306
+ This is the single most important hardening decision: **all backends collapse
307
+ their native failure modes into these three**, so `delivery-queue.ts` needs no
308
+ backend-specific knowledge.
309
+
310
+ ---
311
+
312
+ ## 4. The two backends (proving the contract)
313
+
314
+ ### 4a. Self-hosted SMTP — the default
315
+
316
+ `src/mail/smtp-transport.ts`
317
+
318
+ - **Inbound:** `smtp-server` listener. On `DATA` end → parse (`mailparser`) →
319
+ build `InboundMail` → `await handler(mail)` → only then ACK 250. Handler throw
320
+ → respond 451 (transient) or 550 (permanent) based on the thrown error type.
321
+ - **Outbound:** `nodemailer` configured for direct-to-MX (default) or smarthost
322
+ relay. Maps SMTP responses → `MailSendResult`: 2xx→`delivered`,
323
+ 4xx→`transient`, 5xx→`permanent`.
324
+ - **Capabilities:** `{ outbound: "mx", signsDkim: true (we sign via nodemailer
325
+ DKIM), verifiesInboundAuth: true (mailauth), inbound: "listener",
326
+ maxMessageBytes: configurable }`.
327
+ - Lightweight, zero external service, ideal for dev / internal / allow-listed
328
+ peers.
329
+
330
+ ### 4b. Managed provider — the swap-in (**Postmark first**)
331
+
332
+ `src/mail/provider-transport.ts`
333
+
334
+ Phase 4 targets **Postmark** because its inbound and bounce webhooks map ~1:1
335
+ onto the types we hardened in §3 — least impedance to a passing contract suite,
336
+ and it genuinely exercises the distinct webhook/DSN path (unlike SES, whose
337
+ raw-MIME-in-S3 inbound just re-runs the SMTP parser). Postmark's transactional
338
+ deliverability also matches agent traffic shape.
339
+
340
+ - **Inbound:** Postmark inbound webhook POSTs fully parsed JSON (stripped reply
341
+ text, attachments, SPF/DKIM results) → `InboundMail` → `await handler` →
342
+ respond 200 only after commit; 5xx to make Postmark retry. Postmark's
343
+ **separate bounce webhook** delivers typed bounce records → `InboundBounce`
344
+ (§3) directly.
345
+ - **Outbound:** Postmark REST API; map API responses → `MailSendResult`
346
+ (200 → `delivered`, 4xx throttle/transient → `transient`, hard reject →
347
+ `permanent`).
348
+ - **Capabilities:** `{ outbound: "api", signsDkim: true (provider), inbound:
349
+ "webhook", verifiesInboundAuth: true, ... }`.
350
+
351
+ Both satisfy the *same* interface; the gateway and router are unchanged when
352
+ swapping. You can even run SMTP inbound + provider outbound by composing two
353
+ transports behind a `CompositeMailTransport` (future, not in scope).
354
+
355
+ > **SES remains the documented cost-at-scale alternative.** The interface makes
356
+ > it a config swap, not a rewrite, so "Postmark to prove the contract, SES if
357
+ > per-email cost dominates at high volume" costs nothing later. SES inbound,
358
+ > when added, reuses the SMTP `mailparser` path on raw MIME pulled from S3/SNS.
359
+
360
+ ---
361
+
362
+ ## 4c. Attachment storage — `AttachmentStore` (SQLite default)
363
+
364
+ `src/mail/attachment-store.ts`
365
+
366
+ `MailAttachment.contentRef` is opaque (§3), so attachment bytes live behind a
367
+ small swappable store that mirrors the existing `Storage`-with-two-impls
368
+ pattern:
369
+
370
+ ```ts
371
+ export interface AttachmentStore {
372
+ /** Returns a content-addressed ref (sha256-based). Dedups identical bytes. */
373
+ put(bytes: Buffer, meta: { contentType: string; filename?: string }): Promise<string>;
374
+ get(ref: string): Promise<Buffer>;
375
+ delete(ref: string): Promise<void>;
376
+ }
377
+ ```
378
+
379
+ - **Default: `SqliteAttachmentStore`** — bytes in an `attachments` BLOB table.
380
+ Commits **atomically** with the message (one transaction), single-file backup,
381
+ no orphan GC, and works with the in-memory DB tests already use. Zero-config.
382
+ - **Opt-in: `FsAttachmentStore`** — content-addressed files
383
+ (`<dataDir>/attachments/<sha256>`) for large/high-volume deployments. Keeps the
384
+ DB small, streams big files, avoids blocking the synchronous `better-sqlite3`
385
+ loop. Writes file **before** committing the row, so a crash leaves a GC-able
386
+ orphan file, never a row pointing at missing bytes.
387
+ - **Caps:** configurable per-attachment + total size, tied to
388
+ `capabilities.maxMessageBytes`; over-cap → reject inbound (NACK) or store
389
+ metadata-only (drop bytes, keep the `MailAttachment` record) per policy.
390
+ - Content-addressing by sha256 gives free dedup and dovetails with the inbound
391
+ dedup story (§7b).
392
+
393
+ ---
394
+
395
+ ## 5. The mapper — `email-mapper.ts`
396
+
397
+ Parallels `src/mesh/type-mapper.ts` exactly. Two pure functions:
398
+
399
+ - `inboundMailToMessage(mail, ctx): Message`
400
+ - `from.address` → `sender_id` (the external email, kept verbatim).
401
+ - `to`/`cc` → recipients with `kind`; resolve each to an owning agent via the
402
+ address book (§6); unknown local recipients rejected before commit.
403
+ - `subject` → `subject`.
404
+ - `headers.messageId` → carried in `metadata.rfcMessageId`; **also the dedup
405
+ key** (§7).
406
+ - `headers.inReplyTo` / `references` → `in_reply_to` + `thread_tag`
407
+ (hash of root `References` id) so email threads map onto inbox threads.
408
+ - `authResults`, `remote`, `envelopeFrom` → `metadata` (for audit/abuse).
409
+ - Body → `normalizeContent` (text preferred; html retained in metadata).
410
+ - `messageToOutboundMail(msg, ctx): OutboundMail`
411
+ - Inverse. `Message.id` → `idempotencyKey` and basis for `headers.messageId`.
412
+ - `in_reply_to` → look up parent's stored `rfcMessageId` → `inReplyTo` +
413
+ `references` chain so external clients thread correctly.
414
+
415
+ Reuses `INBOX_META_KEYS`-style stripping so round-tripping doesn't leak control
416
+ fields into user-visible metadata.
417
+
418
+ ---
419
+
420
+ ## 6. Identity / address book — `address-book.ts`
421
+
422
+ Maps between agent ids and email addresses, and declares which domains we own.
423
+
424
+ ```ts
425
+ export interface MailIdentityConfig {
426
+ /** Domains we receive for; inbound RCPT TO outside these is rejected. */
427
+ localDomains: string[];
428
+ /** agent_id → primary email, and reverse. Plus-addressing aware. */
429
+ mappings: Array<{ agentId: string; address: string }>;
430
+ /** Default mailbox for unmatched-but-accepted inbound (e.g. a triage agent). */
431
+ catchAllAgentId?: string;
432
+ }
433
+ ```
434
+
435
+ - Inbound `agent+scope@domain` → agent `agent`, inbox `scope` (reuses existing
436
+ scope semantics).
437
+ - Outbound: an agent's `from` is its mapped address, else a configured default
438
+ `noreply@localDomain`.
439
+ - This is also where the §1 classifier sources its mail-domain list, so
440
+ classification and rendering agree on what "ours" means.
441
+
442
+ ---
443
+
444
+ ## 7. Durability & idempotency (the load-bearing prerequisite)
445
+
446
+ ### 7a. Finish the SQLite delivery queue
447
+
448
+ `delivery-queue.ts` is memory-only today (the `persistence: "sqlite"` config
449
+ value exists but is unimplemented). Mail makes durability mandatory:
450
+
451
+ - Implement a `sqlite` persistence path: `enqueue` / `recordAttempt` /
452
+ `removeEntry` / `flush` / TTL-`tick` all backed by a `mail_queue` table.
453
+ - Survive restart: on boot, reload pending entries and resume retry timers.
454
+ - Default the queue to durable (or hard-warn on memory) whenever mail is enabled.
455
+
456
+ ### 7b. Inbound idempotency (at-least-once → exactly-once-effect)
457
+
458
+ SMTP and webhooks both redeliver. Dedup on `headers.messageId`:
459
+
460
+ - Add a storage method `hasSeenExternalId(id): boolean` + record on commit.
461
+ - Mapper computes a fallback synthetic id (hash of from+date+subject+body) when
462
+ `Message-ID` is absent.
463
+ - Duplicate → ACK success **without** re-injecting (we already have it).
464
+
465
+ ### 7c. Commit-before-ACK
466
+
467
+ Enforced structurally by §2: the backend's ACK is downstream of the awaited
468
+ `onReceive` handler, and the handler only resolves after `storage.putMessage`.
469
+ No backend can ACK early without violating the type contract.
470
+
471
+ ---
472
+
473
+ ## 8. Router & gateway wiring
474
+
475
+ `src/mail/mail-gateway.ts` owns ingress + egress and is the only mail-aware
476
+ component the rest of the system touches.
477
+
478
+ - **Ingress:** constructed with the `MailTransport`; calls
479
+ `transport.onReceive(mail => router.routeMessage(mapped))`. Applies allow-list
480
+ + size + auth-result policy *before* commit; rejects map to handler-throw →
481
+ NACK.
482
+ - **Egress:** `MessageRouter.routeMessage` gains a third branch. After computing
483
+ recipients, partition by `classifyDestination` (§1):
484
+ - `local` → existing inbox path.
485
+ - `federation` → existing `ConnectionManager.route`.
486
+ - `mail` → `MailGateway.send(message)` which renders + enqueues durably, then
487
+ drives `transport.send` with the queue's retry loop.
488
+
489
+ `MessageRouter` gets an optional `setMail(gateway)` mirroring
490
+ `setFederation(...)`, so mail stays opt-in and the core compiles without it.
491
+
492
+ ### New events (mirroring federation events)
493
+
494
+ - `mail.received` — inbound accepted and committed.
495
+ - `mail.sent` — `delivered` disposition.
496
+ - `mail.queued` — transient failure, retry scheduled.
497
+ - `mail.bounced` — permanent failure (outbound `permanent` result *or* inbound
498
+ failed-DSN). Payload carries the original `Message.id` when correlatable.
499
+ - `mail.rejected` — inbound refused (allow-list / size / auth).
500
+ - `mail.config.conflict` — startup warning: a domain is claimed by both a
501
+ federation peer and mail config (§1; federation wins).
502
+
503
+ ### Bounce handling — both paths, config-gated (resolved)
504
+
505
+ A bounce reaches us two ways, and both converge on one handler in the gateway:
506
+
507
+ 1. **Outbound `permanent` result** from `transport.send` (§3).
508
+ 2. **Inbound DSN** — an `InboundMail` with `bounce` set (§3); the gateway
509
+ correlates `bounce.originalMessageId` (or recipient + thread) back to the
510
+ sent `Message` instead of routing it as a new inbox message.
511
+
512
+ For both, gateway config `bounce` controls the response:
513
+
514
+ ```ts
515
+ bounce?: {
516
+ /** Always emit the mail.bounced event (cheap, for observability). default true */
517
+ emitEvent?: boolean;
518
+ /** Inject a synthetic bounce Message into the original sender's inbox so the
519
+ * agent sees the failure inline (in_reply_to = original, importance: high).
520
+ * default true */
521
+ synthesizeInboxMessage?: boolean;
522
+ };
523
+ ```
524
+
525
+ Default is **both** on. The synthesized message reuses the existing `Message`
526
+ shape (`in_reply_to` = the failed message, `subject` = "Delivery failed: …",
527
+ `metadata.bounce` = the `InboundBounce`/result detail), so threading and
528
+ traceability pick it up with no special-casing. `delayed` DSNs do **not**
529
+ synthesize — they map to `mail.queued` since the peer MTA is still retrying.
530
+
531
+ ---
532
+
533
+ ## 9. Security posture (interface-level, not optional)
534
+
535
+ - **Inbound is untrusted.** Allow-list senders/domains; rate-limit per peer IP;
536
+ enforce `maxMessageBytes` before buffering; cap attachment count/size.
537
+ - **Surface auth verdicts.** `authResults` flows into metadata; gateway policy
538
+ can drop or quarantine `dmarc: "fail"`.
539
+ - **Bodies feed agents.** Treat inbound text/html as untrusted external data
540
+ (same posture as the webhook/PR-body guidance) — never auto-execute
541
+ instructions found in email.
542
+ - **Outbound deliverability caveat (self-hosted).** Reaching Gmail/Outlook needs
543
+ SPF/DKIM/DMARC + rDNS + IP reputation. The interface lets you keep self-hosted
544
+ as default and swap `provider-transport` for outbound when you need the big
545
+ mailboxes — no core changes. Documented, not solved in code.
546
+ - **Secrets** (provider API keys, DKIM private keys) via env/config, never
547
+ committed; mirror existing `InboxConfig.map.auth` patterns.
548
+
549
+ ---
550
+
551
+ ## 10. Config surface
552
+
553
+ Extend `InboxConfig` (src/types.ts):
554
+
555
+ ```ts
556
+ mail?: {
557
+ enabled: boolean;
558
+ backend: "smtp" | "provider";
559
+ identity: MailIdentityConfig; // localDomains + agent↔address mappings (§6)
560
+ /** Domains we will SEND to without erroring (§1 rule 3). Receiving uses
561
+ * identity.localDomains. Anything outside both → UnknownDestinationError. */
562
+ routableDomains?: string[];
563
+ smtp?: { listenPort?: number; relay?: { host: string; port: number; auth?: {...} };
564
+ dkim?: { domain: string; selector: string; privateKeyRef: string } };
565
+ provider?: { name: "ses" | "postmark" | "mailgun"; apiKeyRef: string;
566
+ webhookPath?: string };
567
+ queue?: Partial<DeliveryQueueConfig>; // defaults to sqlite when mail enabled
568
+ bounce?: { emitEvent?: boolean; synthesizeInboxMessage?: boolean }; // §8, both default true
569
+ };
570
+ ```
571
+
572
+ Federation peers gain an optional `domains` field so they can claim dotted
573
+ addresses for the §1 classifier:
574
+
575
+ ```ts
576
+ export interface FederationPeerConfig {
577
+ systemId: string;
578
+ domains?: string[]; // NEW: dotted addresses routed to this peer, not mail
579
+ url?: string;
580
+ meshPeerId?: string;
581
+ // …existing fields…
582
+ }
583
+ ```
584
+
585
+ `src/index.ts` wires: if `mail.enabled`, construct the backend per `backend`,
586
+ build `MailGateway`, call `router.setMail(gateway)`, and register inbound.
587
+
588
+ ---
589
+
590
+ ## 11. Test plan (vitest, in-process — no real sockets/network)
591
+
592
+ Follow the existing `MockMapServer` / `MockMeshContext` convention with a
593
+ `MockMailTransport` (in-memory loopback implementing `MailTransport`).
594
+
595
+ - `test/mail/destination.test.ts` — §1 classifier: local vs federation vs mail,
596
+ the `corp.internal` peer-wins case, dotted-domain detection, ordering.
597
+ - `test/mail/email-mapper.test.ts` — round-trip; threading header ↔ thread_tag;
598
+ plus-addressing → scope; metadata stripping; missing Message-ID fallback id.
599
+ - `test/mail/mail-transport-contract.test.ts` — a **shared contract suite** run
600
+ against `MockMailTransport` (and later each real backend): send idempotency on
601
+ duplicate key; `send` returns (never throws) on transient/permanent; ACK only
602
+ after handler resolves; handler-throw → NACK; `start` resolves when ready.
603
+ - `test/mail/delivery-queue-sqlite.test.ts` — durable enqueue/flush/retry/TTL;
604
+ survives a simulated restart.
605
+ - `test/mail/inbound-dedup.test.ts` — redelivered Message-ID committed once.
606
+ - `test/mail/mail-gateway.test.ts` — ingress allow-list/size/auth rejection;
607
+ egress partitioning of mixed local+federation+mail recipients.
608
+ - `test/mail/integration.test.ts` — agent ↔ external loopback both directions
609
+ via `MockMailTransport`.
610
+
611
+ The shared contract suite is the real hardening payoff: any backend we add later
612
+ must pass the *same* tests, so the interface stays honest.
613
+
614
+ ---
615
+
616
+ ## 12. Phasing — ✅ complete
617
+
618
+ 1. ✅ **Foundations:** §1 classifier + §7a SQLite queue + types (§2, §3) + config
619
+ (§10). No behavior change; everything compiles and tests green.
620
+ 2. ✅ **Mapper + address book** (§5, §6) + `MockMailTransport` + contract suite
621
+ (§11). Interface proven against a fake before any I/O.
622
+ 3. ✅ **Self-hosted SMTP backend** (§4a) + gateway wiring (§8, incl. both bounce
623
+ paths) + dedup (§7b/c). Default path working end-to-end, real-socket e2e.
624
+ 4. ✅ **Managed provider backend** (§4b, Postmark) — proves the swap; same
625
+ contract suite via a fake client. `AttachmentStore` SQLite default landed in
626
+ phase 2.
627
+ 5. ✅ **Hardening pass:** inbound rate limiting + abuse controls (attachment cap,
628
+ allow-list, DMARC), DSN parsing for the SMTP backend (`parseDsnFromParsed`),
629
+ docs in CLAUDE.md, plus a post-review pass (retry re-entrancy, bounce-replay
630
+ dedup, DSN false-positive fix, Postmark error classification, webhook
631
+ body-cap + basic-auth, address/attachment validation).
632
+ 6. ✅ **`FsAttachmentStore`:** opt-in filesystem attachment backend behind the
633
+ same `AttachmentStore` interface (content-addressed, atomic write-then-rename,
634
+ sharded, path-traversal-safe, with GC). Both backends pass a shared contract.
635
+
636
+ Phases 1–2 are where the interface was locked; everything after was "fill in a
637
+ backend that passes the contract suite."
638
+
639
+ ---
640
+
641
+ ## Resolved decisions
642
+
643
+ 1. **Classification (§1):** explicit-list only. Mail domains and federation
644
+ peer domains are both declared in config; an unknown `@x` is an
645
+ `UnknownDestinationError`, never a guess. Federation peers may claim
646
+ dotted domains, federation wins on conflict (with a `mail.config.conflict`
647
+ warning).
648
+ 2. **Bounce (§8):** both `emitEvent` and `synthesizeInboxMessage`, config-gated,
649
+ both defaulting on; inbound DSNs and outbound `permanent` results converge on
650
+ the same handler.
651
+ 3. **Provider (§4b):** **Postmark first** — inbound + typed bounce webhooks map
652
+ ~1:1 to our types, fastest contract proof. SES kept as documented
653
+ cost-at-scale alternative.
654
+ 4. **Attachments (§4c):** **`AttachmentStore` interface, SQLite BLOB default**
655
+ (atomic, zero-config, test-friendly); `FsAttachmentStore` opt-in for
656
+ large/high-volume. Content-addressed, size-capped.
657
+
658
+ ## Open questions for review
659
+
660
+ *(none blocking — interface is locked; ready to build Phase 1.)*