@serve.zone/dcrouter 18.1.1 → 18.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -49,6 +49,32 @@ import type { IEmailOutboundEgressStatus, IEmailPortConfig, IEmailServerSettings
49
49
  import type { IDcRouterRouteConfig, IRemoteIngressHubSettings, IRemoteIngressPerformanceConfig, TRemoteIngressHubSettingsUpdate } from '../ts_interfaces/data/remoteingress.js';
50
50
  import type { ISecurityCompiledPolicy } from '../ts_interfaces/data/security-policy.js';
51
51
 
52
+ /**
53
+ * dcrouter's superset of SmartMTA's SMTP TLS options.
54
+ *
55
+ * SmartMTA consumes PEM content (`certPem`/`keyPem`). Deployments configure
56
+ * certificate FILES, so dcrouter accepts the path form too and loads it eagerly
57
+ * at startup — a path that cannot be read fails startup rather than silently
58
+ * leaving the listener without TLS.
59
+ */
60
+ export interface IDcRouterEmailTlsConfig extends NonNullable<IUnifiedEmailServerOptions['tls']> {
61
+ /** Path to the certificate chain PEM file. Requires keyPath. */
62
+ certPath?: string;
63
+ /** Path to the private key PEM file. Requires certPath. */
64
+ keyPath?: string;
65
+ }
66
+
67
+ /** dcrouter's email server options: SmartMTA's, with the path-shaped tls block. */
68
+ export type IDcRouterEmailConfig = Omit<IUnifiedEmailServerOptions, 'tls'> & {
69
+ tls?: IDcRouterEmailTlsConfig;
70
+ };
71
+
72
+ /** Whether an email configuration carries any SMTP AUTH credentials. */
73
+ const emailConfigHasAuth = (emailConfigArg: IUnifiedEmailServerOptions | undefined): boolean =>
74
+ !!emailConfigArg?.auth?.required
75
+ || !!emailConfigArg?.auth?.users?.length
76
+ || !!emailConfigArg?.auth?.accounts?.length;
77
+
52
78
  export interface IDcRouterOptions {
53
79
  /** Base directory for all dcrouter data. Defaults to ~/.serve.zone/dcrouter */
54
80
  baseDir?: string;
@@ -66,7 +92,7 @@ export interface IDcRouterOptions {
66
92
  * Email server configuration
67
93
  * This enables all email handling with pattern-based routing
68
94
  */
69
- emailConfig?: IUnifiedEmailServerOptions;
95
+ emailConfig?: IDcRouterEmailConfig;
70
96
 
71
97
  /** SmartBucket configuration for durable SmartMTA queue and attachment blobs. */
72
98
  emailBlobStorage?: ISmartMtaBlobStorageConfig;
@@ -1623,6 +1649,10 @@ export class DcRouter {
1623
1649
  expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
1624
1650
  source: event.source,
1625
1651
  });
1652
+ // Renewals arrive here too. The SMTP listener holds its PEM material as
1653
+ // listener-level Rust configuration, so a renewed mail-hostname
1654
+ // certificate must be pushed or STARTTLS keeps serving the stale one.
1655
+ void this.reapplyEmailTlsMaterial(event.domain);
1626
1656
  });
1627
1657
 
1628
1658
  // Note: smartproxy v27.5.0 emits only 'certificate-issued' and 'certificate-failed'.
@@ -1955,6 +1985,10 @@ export class DcRouter {
1955
1985
  storageManager: this.smartMtaBlobStorageManager,
1956
1986
  };
1957
1987
 
1988
+ const emailTls = await this.resolveEmailTlsMaterial();
1989
+ const tlsTerminatedPorts = this.resolveEdgeTerminatedEmailPorts(portMapping);
1990
+ this.logEmailAuthTransportPosture(emailConfigHasAuth(baseEmailConfig), emailTls, mappedEmailPorts, tlsTerminatedPorts, mappedSecurePort);
1991
+
1958
1992
  let emailConfig: IUnifiedEmailServerOptions = await this.smtpAccountManager.composeEmailConfig({
1959
1993
  ...this.options.emailConfig,
1960
1994
  ports: mappedEmailPorts,
@@ -1962,6 +1996,7 @@ export class DcRouter {
1962
1996
  dkimKeyProvisioning: 'caller-managed',
1963
1997
  persistRoutes: this.options.emailConfig.persistRoutes ?? false,
1964
1998
  queue: queueOptions,
1999
+ ...(emailTls ? { tls: { ...baseEmailConfig.tls, certPem: emailTls.certPem, keyPem: emailTls.keyPem } } : {}),
1965
2000
  outbound: {
1966
2001
  ...baseEmailConfig.outbound,
1967
2002
  connectionProxyProvider: (context) => this.mailEgressCoordinator.provideConnectionProxy(context),
@@ -1971,6 +2006,7 @@ export class DcRouter {
1971
2006
  ...(mappedSecurePort !== undefined
1972
2007
  ? { securePort: mappedSecurePort }
1973
2008
  : {}),
2009
+ ...(tlsTerminatedPorts.length > 0 ? { tlsTerminatedPorts } : {}),
1974
2010
  recipientValidation: true,
1975
2011
  proxyProtocol: {
1976
2012
  ...baseEmailConfig.smtp?.proxyProtocol,
@@ -2178,6 +2214,168 @@ export class DcRouter {
2178
2214
  this.mailDnsSync.requestSync('email server start');
2179
2215
  }
2180
2216
 
2217
+ /**
2218
+ * Resolve the PEM material for the SMTP listener.
2219
+ *
2220
+ * SmartMTA consumes `tls.certPem`/`tls.keyPem`; a path-shaped `tls` block is
2221
+ * ignored, which leaves the Rust listener with no TLS material at all — no
2222
+ * STARTTLS on the plain submission ports. Resolution order matches the
2223
+ * RemoteIngress tunnel precedent: explicit paths, then the ACME cert store.
2224
+ *
2225
+ * Explicitly configured paths that cannot be read FAIL STARTUP. Silently
2226
+ * continuing without TLS is what produced a cleartext submission port in
2227
+ * production, so a broken explicit configuration must be impossible to miss.
2228
+ */
2229
+ private async resolveEmailTlsMaterial(): Promise<{ certPem: string; keyPem: string; source: string } | undefined> {
2230
+ const tlsConfig = this.options.emailConfig?.tls as (IUnifiedEmailServerOptions['tls'] & {
2231
+ certPath?: string;
2232
+ keyPath?: string;
2233
+ }) | undefined;
2234
+
2235
+ if (tlsConfig?.certPem && tlsConfig?.keyPem) {
2236
+ return { certPem: tlsConfig.certPem, keyPem: tlsConfig.keyPem, source: 'inline PEM' };
2237
+ }
2238
+
2239
+ if (tlsConfig?.certPath || tlsConfig?.keyPath) {
2240
+ if (!tlsConfig.certPath || !tlsConfig.keyPath) {
2241
+ throw new Error(
2242
+ 'emailConfig.tls requires both certPath and keyPath when either is configured',
2243
+ );
2244
+ }
2245
+ let certPem: string;
2246
+ let keyPem: string;
2247
+ try {
2248
+ certPem = await plugins.fs.promises.readFile(tlsConfig.certPath, 'utf8');
2249
+ keyPem = await plugins.fs.promises.readFile(tlsConfig.keyPath, 'utf8');
2250
+ } catch (error: unknown) {
2251
+ throw new Error(
2252
+ `Unable to read the configured SMTP TLS material (certPath=${tlsConfig.certPath}, keyPath=${tlsConfig.keyPath}): ${(error as Error).message}`,
2253
+ );
2254
+ }
2255
+ if (!certPem.trim() || !keyPem.trim()) {
2256
+ throw new Error(
2257
+ `The configured SMTP TLS material is empty (certPath=${tlsConfig.certPath}, keyPath=${tlsConfig.keyPath})`,
2258
+ );
2259
+ }
2260
+ logger.log('info', `SMTP TLS material loaded from configured paths (${tlsConfig.certPath})`);
2261
+ return { certPem, keyPem, source: `configured paths (${tlsConfig.certPath})` };
2262
+ }
2263
+
2264
+ const mailHostname = this.options.emailConfig?.hostname;
2265
+ if (mailHostname) {
2266
+ try {
2267
+ const stored = await ProxyCertDoc.findByDomain(mailHostname);
2268
+ if (stored?.publicKey && stored?.privateKey) {
2269
+ logger.log('info', `SMTP TLS material loaded from the stored ACME certificate for ${mailHostname}`);
2270
+ return { certPem: stored.publicKey, keyPem: stored.privateKey, source: `stored ACME certificate for ${mailHostname}` };
2271
+ }
2272
+ } catch (error: unknown) {
2273
+ logger.log('warn', `Unable to read the stored certificate for the mail hostname ${mailHostname}: ${(error as Error).message}`);
2274
+ }
2275
+ }
2276
+
2277
+ return undefined;
2278
+ }
2279
+
2280
+ /**
2281
+ * Push renewed TLS material to the running SMTP listener.
2282
+ *
2283
+ * Only reacts to the mail hostname, and only when explicit paths are NOT
2284
+ * configured — an operator-managed file pair is not superseded by an ACME
2285
+ * renewal for the same name. SmartMTA restarts just the Rust listener when
2286
+ * the PEM material actually changes.
2287
+ */
2288
+ private async reapplyEmailTlsMaterial(domainArg: string): Promise<void> {
2289
+ const emailServer = this.emailServer;
2290
+ const mailHostname = this.options.emailConfig?.hostname;
2291
+ if (!emailServer || !mailHostname) return;
2292
+ if (domainArg.toLowerCase() !== mailHostname.toLowerCase()) return;
2293
+ const tlsConfig = this.options.emailConfig?.tls;
2294
+ if (tlsConfig?.certPath || tlsConfig?.keyPath) return;
2295
+
2296
+ try {
2297
+ const emailTls = await this.resolveEmailTlsMaterial();
2298
+ if (!emailTls) return;
2299
+ emailServer.updateOptions({
2300
+ tls: { ...this.options.emailConfig?.tls, certPem: emailTls.certPem, keyPem: emailTls.keyPem },
2301
+ });
2302
+ if (this.options.emailConfig) {
2303
+ this.options.emailConfig.tls = {
2304
+ ...this.options.emailConfig.tls,
2305
+ certPem: emailTls.certPem,
2306
+ keyPem: emailTls.keyPem,
2307
+ };
2308
+ }
2309
+ logger.log('info', `Pushed renewed SMTP TLS material for ${mailHostname} to the email listener`);
2310
+ } catch (error: unknown) {
2311
+ logger.log('error', `Unable to apply renewed SMTP TLS material for ${mailHostname}: ${(error as Error).message}`);
2312
+ }
2313
+ }
2314
+
2315
+ /**
2316
+ * Internal SMTP ports whose public leg is TLS-terminated by CoreTraffic.
2317
+ *
2318
+ * Those backend legs arrive as plaintext even though the client's channel was
2319
+ * encrypted, so without declaring them the AUTH-requires-encryption gate would
2320
+ * refuse authentication on the only encrypted submission port. Derived from
2321
+ * the generated route set rather than hardcoding 465, so it stays true when
2322
+ * emailPortConfig changes.
2323
+ */
2324
+ private resolveEdgeTerminatedEmailPorts(portMapping: Record<number, number>): number[] {
2325
+ if (!this.options.emailConfig) return [];
2326
+ const terminatedPorts: number[] = [];
2327
+ for (const route of this.emailRouteBuilder.generateEmailRoutes(this.options.emailConfig)) {
2328
+ if ((route.action as { tls?: { mode?: string } }).tls?.mode !== 'terminate') continue;
2329
+ const publicPorts = Array.isArray(route.match?.ports) ? route.match.ports : [];
2330
+ for (const publicPort of publicPorts) {
2331
+ if (typeof publicPort !== 'number') continue;
2332
+ const internalPort = portMapping[publicPort] || publicPort + 10000;
2333
+ // The implicit-TLS securePort is terminated by smartmta itself.
2334
+ if (internalPort === this.options.emailConfig.smtp?.securePort) continue;
2335
+ if (!terminatedPorts.includes(internalPort)) {
2336
+ terminatedPorts.push(internalPort);
2337
+ }
2338
+ }
2339
+ }
2340
+ return terminatedPorts;
2341
+ }
2342
+
2343
+ /**
2344
+ * State the transport posture of every SMTP listener port at startup.
2345
+ *
2346
+ * AUTH is only offered on an encrypted transport, so an operator has to be
2347
+ * able to see at a glance which submission ports can actually authenticate —
2348
+ * missing certificate material silently disabling AUTH on 587 would otherwise
2349
+ * look like a client problem.
2350
+ */
2351
+ private logEmailAuthTransportPosture(
2352
+ authConfigured: boolean,
2353
+ emailTls: { source: string } | undefined,
2354
+ mappedPorts: number[],
2355
+ tlsTerminatedPorts: number[],
2356
+ mappedSecurePort: number | undefined,
2357
+ ): void {
2358
+ const authCapable: number[] = [];
2359
+ const cleartext: number[] = [];
2360
+ for (const port of mappedPorts) {
2361
+ if (port === mappedSecurePort || tlsTerminatedPorts.includes(port) || emailTls) {
2362
+ authCapable.push(port);
2363
+ } else {
2364
+ cleartext.push(port);
2365
+ }
2366
+ }
2367
+ logger.log(
2368
+ 'info',
2369
+ `SMTP transport posture: TLS material=${emailTls ? emailTls.source : 'NONE'}, edge-terminated ports=[${tlsTerminatedPorts.join(', ') || 'none'}], implicit-TLS port=${mappedSecurePort ?? 'none'}, AUTH-capable ports=[${authCapable.join(', ') || 'none'}]`,
2370
+ );
2371
+ if (authConfigured && !emailTls && cleartext.length > 0) {
2372
+ logger.log(
2373
+ 'error',
2374
+ `SMTP AUTH is configured but ports [${cleartext.join(', ')}] have no TLS material and no upstream TLS terminator — AUTH is refused there because credentials must never cross a cleartext channel. Configure emailConfig.tls.certPath/keyPath, or provision an ACME certificate for ${this.options.emailConfig?.hostname || 'the mail hostname'}.`,
2375
+ );
2376
+ }
2377
+ }
2378
+
2181
2379
  /**
2182
2380
  * Readiness of the RemoteIngress outbound mail egress path (mail-tagged edges).
2183
2381
  */
@@ -8,10 +8,13 @@ import type {
8
8
  import { AcceptEnvelopeRejectionError } from '@push.rocks/smartmta';
9
9
  import type {
10
10
  Email,
11
+ IAcceptedEnvelopeDispatchMetadata,
12
+ IAcceptedEnvelopeRecipientPlan,
11
13
  IAcceptEnvelopeContext,
12
14
  IExtendedSmtpSession,
13
15
  IMessageAcceptanceContext,
14
16
  IMessageAcceptanceDecision,
17
+ IResolvedRecipientRoute,
15
18
  UnifiedEmailServer,
16
19
  } from '@push.rocks/smartmta';
17
20
  import type { DcRouter } from '../classes.dcrouter.js';
@@ -76,6 +79,9 @@ const ACCEPTED_EMAIL_SPOOL_BATCH_SIZE = 25;
76
79
  const ACCEPTED_EMAIL_STOP_DRAIN_TIMEOUT_MS = 30_000;
77
80
  /** Retention for catch-all stored inbound mail (30 days). */
78
81
  const INBOUND_STORE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
82
+ /** Redispatch attempts for the non-store recipients of a durable envelope. */
83
+ const ENVELOPE_DISPATCH_MAX_ATTEMPTS = 10;
84
+ const ENVELOPE_DISPATCH_RETRY_DELAY_MS = 5 * 60_000;
79
85
 
80
86
  /**
81
87
  * Permanent per-email storage failure: the raw RFC822 payload of an accepted
@@ -136,8 +142,32 @@ type TStoredCachedEmailSession = {
136
142
  };
137
143
  };
138
144
 
145
+ /**
146
+ * Persisted per-recipient dispatch state for a durably accepted envelope.
147
+ * Holds everything needed to replay `dispatchAcceptedEnvelope` byte-identically:
148
+ * upstream fingerprints each recipient over the raw message, the metadata and
149
+ * the plan entry, so a replay reconstructed from anything else would be refused
150
+ * as an idempotency-key reuse instead of retrying.
151
+ */
152
+ type TStoredEnvelopeDispatch = {
153
+ plan: IAcceptedEnvelopeRecipientPlan[];
154
+ metadata: IAcceptedEnvelopeDispatchMetadata;
155
+ attempts: number;
156
+ results?: Array<{
157
+ recipient: string;
158
+ status: string;
159
+ message?: string;
160
+ smtpCode?: number;
161
+ }>;
162
+ pendingRecipients?: string[];
163
+ /** Set once retries are exhausted; the row stops being redispatched. */
164
+ abandonedAt?: string;
165
+ };
166
+
139
167
  type TStoredCachedEmailRouteData = {
168
+ acceptance?: string;
140
169
  session?: TStoredCachedEmailSession;
170
+ envelopeDispatch?: TStoredEnvelopeDispatch;
141
171
  smartMta?: {
142
172
  status?: 'queued' | 'deferred' | 'delivered' | 'failed';
143
173
  nextAttempt?: string;
@@ -185,6 +215,54 @@ export class AcceptedEmailSpool {
185
215
 
186
216
  constructor(private dcRouterRef: DcRouter) {}
187
217
 
218
+ /**
219
+ * Direction is decided by the RECIPIENT, never by whether the session
220
+ * authenticated. Authentication grants permission to relay; it does not make
221
+ * a message addressed to a mailbox we host into outbound mail. A message with
222
+ * at least one locally hosted recipient is inbound — a local mailbox receives
223
+ * it — and only an envelope addressed exclusively to remote recipients is
224
+ * outbound.
225
+ *
226
+ * `IResolvedRecipientRoute.localDomain` is smartmta's own per-recipient
227
+ * verdict (non-null exactly when the domain registry hosts the domain), so
228
+ * the classification uses the same truth the routing decision used. When an
229
+ * acceptance context carries no resolution the envelope recipients are
230
+ * classified against the live registry instead — never against the session,
231
+ * which is the mistake being fixed.
232
+ */
233
+ private deriveDirectionFromResolvedRoutes(
234
+ resolvedRecipientRoutes: readonly IResolvedRecipientRoute[] | undefined,
235
+ envelopeRecipientsArg: readonly string[],
236
+ ): TCachedEmailDirection {
237
+ if (resolvedRecipientRoutes?.length) {
238
+ return resolvedRecipientRoutes.some((resolution) => !!resolution.localDomain)
239
+ ? 'inbound'
240
+ : 'outbound';
241
+ }
242
+ return this.deriveDirectionFromRecipients(envelopeRecipientsArg);
243
+ }
244
+
245
+ /**
246
+ * Recipient-derived direction for programmatic submitters, which have no SMTP
247
+ * recipient resolution to consult. Falls back to the live domain registry.
248
+ */
249
+ private deriveDirectionFromRecipients(recipientsArg: readonly string[]): TCachedEmailDirection {
250
+ const domainRegistry = this.dcRouterRef.emailServer?.domainRegistry;
251
+ if (!domainRegistry) {
252
+ // Without the registry there is no recipient truth to classify against.
253
+ // Programmatic submission is relay by construction, so outbound is the
254
+ // honest label rather than guessing from the session.
255
+ return 'outbound';
256
+ }
257
+ for (const recipient of recipientsArg) {
258
+ const domain = recipient.split('@')[1]?.trim().toLowerCase();
259
+ if (domain && domainRegistry.isDomainRegistered(domain)) {
260
+ return 'inbound';
261
+ }
262
+ }
263
+ return 'outbound';
264
+ }
265
+
188
266
  public async acceptMessage(
189
267
  context: IMessageAcceptanceContext,
190
268
  processAfterAccept = true,
@@ -218,7 +296,10 @@ export class AcceptedEmailSpool {
218
296
  await this.persistRawMessage(cachedEmail, persistedRawMessage);
219
297
  cachedEmail.status = 'pending';
220
298
  cachedEmail.nextAttempt = new Date();
221
- cachedEmail.direction = session.authenticated ? 'outbound' : 'inbound';
299
+ cachedEmail.direction = this.deriveDirectionFromResolvedRoutes(
300
+ context.resolvedRecipientRoutes,
301
+ envelopeRecipients.length > 0 ? envelopeRecipients : cachedEmail.to,
302
+ );
222
303
  cachedEmail.acceptedAt = Date.now();
223
304
  cachedEmail.routeData = JSON.stringify({
224
305
  acceptedAt: new Date().toISOString(),
@@ -288,7 +369,7 @@ export class AcceptedEmailSpool {
288
369
  cachedEmail.status = 'pending';
289
370
  cachedEmail.nextAttempt = new Date();
290
371
  cachedEmail.direction = optionsArg.direction
291
- ?? (optionsArg.session.authenticated ? 'outbound' : 'inbound');
372
+ ?? this.deriveDirectionFromRecipients(optionsArg.envelope.rcptTo);
292
373
  cachedEmail.acceptedAt = Date.now();
293
374
  cachedEmail.routeData = JSON.stringify({
294
375
  acceptedAt: new Date().toISOString(),
@@ -379,7 +460,10 @@ export class AcceptedEmailSpool {
379
460
  cachedEmail.id,
380
461
  );
381
462
  await this.persistRawMessage(cachedEmail, persistedRawMessage);
382
- cachedEmail.direction = session.authenticated ? 'outbound' : 'inbound';
463
+ cachedEmail.direction = this.deriveDirectionFromResolvedRoutes(
464
+ context.resolvedRecipientRoutes,
465
+ context.envelope.rcptTo,
466
+ );
383
467
  cachedEmail.acceptedAt = startedAtMs;
384
468
 
385
469
  if (acceptance.dmarcReject) {
@@ -415,6 +499,15 @@ export class AcceptedEmailSpool {
415
499
  } else {
416
500
  cachedEmail.status = 'accepted';
417
501
  }
502
+ if (nonStoreEntries.length > 0) {
503
+ // Relay recipients are dispatched after this row is committed. Park the
504
+ // row in a status the spool actually scans so a crash between the commit
505
+ // and the dispatch leaves those recipients recoverable instead of stranded
506
+ // in a terminal-looking 'accepted' row. The dispatch outcome settles it
507
+ // straight back to the acceptance status.
508
+ cachedEmail.status = 'deferred';
509
+ cachedEmail.nextAttempt = new Date(Date.now() + ENVELOPE_DISPATCH_RETRY_DELAY_MS);
510
+ }
418
511
  cachedEmail.deliveredAt = new Date();
419
512
  cachedEmail.setTTL(INBOUND_STORE_RETENTION_MS);
420
513
  cachedEmail.appendSmtpTransaction(this.buildInboundTransaction(context, cachedEmail.id, {
@@ -424,6 +517,7 @@ export class AcceptedEmailSpool {
424
517
  doubts: acceptance.doubts,
425
518
  startedAtMs,
426
519
  }));
520
+ const dispatchMetadata = this.buildEnvelopeDispatchMetadata(context);
427
521
  cachedEmail.routeData = JSON.stringify({
428
522
  acceptedAt: new Date(startedAtMs).toISOString(),
429
523
  acceptance: 'durable-envelope',
@@ -434,6 +528,18 @@ export class AcceptedEmailSpool {
434
528
  source: entry.source,
435
529
  actionType: entry.action.type,
436
530
  })),
531
+ // Persisted BEFORE the SMTP 250, so a dispatch failure for the non-store
532
+ // recipients is recoverable instead of being lost with the process.
533
+ ...(nonStoreEntries.length > 0
534
+ ? {
535
+ envelopeDispatch: {
536
+ plan: [...plan],
537
+ metadata: dispatchMetadata,
538
+ attempts: 0,
539
+ pendingRecipients: nonStoreEntries.map((entry) => entry.recipient),
540
+ } satisfies TStoredEnvelopeDispatch,
541
+ }
542
+ : {}),
437
543
  session: {
438
544
  id: session.id,
439
545
  remoteAddress: session.remoteAddress,
@@ -461,28 +567,193 @@ export class AcceptedEmailSpool {
461
567
  }
462
568
 
463
569
  if (nonStoreEntries.length > 0) {
464
- const dispatchResult = await emailServer.dispatchAcceptedEnvelope(
465
- context.rawMessage,
466
- plan,
467
- {
468
- mailFrom: context.envelope.mailFrom,
469
- session: {
470
- id: session.id,
471
- remoteAddress: session.remoteAddress,
472
- clientHostname: session.clientHostname,
473
- secure: !!session.secure,
474
- authenticated: !!session.authenticated,
475
- },
476
- },
477
- );
478
- const failed = dispatchResult.results.filter(
479
- (result) => result.status === 'failed' || result.status === 'rejected',
570
+ await this.dispatchEnvelopeRecipients(cachedEmail, persistedRawMessage, emailServer);
571
+ }
572
+ this.trackAcceptedInboundEmail(cachedEmail);
573
+ }
574
+
575
+ /**
576
+ * Session metadata for `dispatchAcceptedEnvelope`. It feeds the per-recipient
577
+ * idempotency fingerprint, so it must be JSON-round-trip stable: a replay
578
+ * reconstructed from persisted state has to produce byte-identical metadata
579
+ * or upstream refuses it as an idempotency-key reuse.
580
+ */
581
+ private buildEnvelopeDispatchMetadata(
582
+ context: IAcceptEnvelopeContext,
583
+ ): IAcceptedEnvelopeDispatchMetadata {
584
+ const session = context.session;
585
+ return {
586
+ mailFrom: context.envelope.mailFrom,
587
+ session: {
588
+ id: session.id || '',
589
+ remoteAddress: session.remoteAddress || '',
590
+ clientHostname: session.clientHostname || '',
591
+ secure: !!session.secure,
592
+ authenticated: !!session.authenticated,
593
+ },
594
+ };
595
+ }
596
+
597
+ /**
598
+ * Dispatch (or redispatch) the non-store recipients of a durably accepted
599
+ * envelope.
600
+ *
601
+ * The raw bytes handed to upstream are the exact bytes persisted for this row,
602
+ * because upstream fingerprints every recipient over the raw message: a later
603
+ * replay with different bytes would be rejected as an idempotency-key reuse
604
+ * rather than retried. Recipients that already succeeded are short-circuited
605
+ * by upstream's checkpoints, and `failed` results are deliberately not
606
+ * checkpointed upstream, so an identical replay retries exactly those.
607
+ *
608
+ * A `failed` recipient leaves the row non-terminal so the spool retries it;
609
+ * a `rejected` recipient is a permanent per-recipient refusal and is recorded
610
+ * durably instead of being retried. Either way the outcome is persisted — a
611
+ * relay recipient is never silently dropped after the SMTP 250.
612
+ */
613
+ private async dispatchEnvelopeRecipients(
614
+ cachedEmailArg: CachedEmail,
615
+ rawMessageArg: string,
616
+ emailServerArg: UnifiedEmailServer,
617
+ ): Promise<void> {
618
+ const routeData = this.parseCachedEmailRouteData(cachedEmailArg);
619
+ const envelopeDispatch = routeData.envelopeDispatch;
620
+ if (!envelopeDispatch || envelopeDispatch.abandonedAt) return;
621
+
622
+ const attempts = (envelopeDispatch.attempts || 0) + 1;
623
+ let dispatchResults: Array<{
624
+ recipient: string;
625
+ status: string;
626
+ message?: string;
627
+ smtpCode?: number;
628
+ }>;
629
+ try {
630
+ const dispatchResult = await emailServerArg.dispatchAcceptedEnvelope(
631
+ plugins.buffer.Buffer.from(rawMessageArg, 'utf8'),
632
+ envelopeDispatch.plan,
633
+ envelopeDispatch.metadata,
480
634
  );
481
- if (failed.length > 0) {
482
- logger.log('warn', `Durable envelope ${cachedEmail.id}: ${failed.length}/${dispatchResult.results.length} non-store recipients failed dispatch: ${failed.map((result) => `${result.recipient}=${result.message || result.status}`).join('; ')}`);
635
+ dispatchResults = dispatchResult.results.map((result) => ({
636
+ recipient: result.recipient,
637
+ status: result.status,
638
+ ...(result.message ? { message: result.message } : {}),
639
+ ...(result.smtpCode !== undefined ? { smtpCode: result.smtpCode } : {}),
640
+ }));
641
+ } catch (error: unknown) {
642
+ // A whole-call failure (missing managed queue storage, malformed
643
+ // checkpoint) is retried the same way a per-recipient failure is.
644
+ dispatchResults = envelopeDispatch.plan
645
+ .filter((entry) => entry.action.type !== 'store' && entry.action.type !== 'deliver')
646
+ .map((entry) => ({
647
+ recipient: entry.recipient,
648
+ status: 'failed',
649
+ message: (error as Error).message,
650
+ }));
651
+ }
652
+
653
+ const retryable = dispatchResults.filter((result) => result.status === 'failed');
654
+ const rejected = dispatchResults.filter((result) => result.status === 'rejected');
655
+ const exhausted = retryable.length > 0 && attempts >= ENVELOPE_DISPATCH_MAX_ATTEMPTS;
656
+
657
+ await this.persistEnvelopeDispatchOutcome(cachedEmailArg.id, {
658
+ attempts,
659
+ results: dispatchResults,
660
+ pendingRecipients: exhausted ? [] : retryable.map((result) => result.recipient),
661
+ abandoned: exhausted,
662
+ });
663
+
664
+ if (rejected.length > 0) {
665
+ logger.log('error', `Durable envelope ${cachedEmailArg.id}: ${rejected.length} relay recipient(s) permanently refused: ${rejected.map((result) => `${result.recipient}=${result.smtpCode || 550} ${result.message || 'rejected'}`).join('; ')}`);
666
+ }
667
+ if (exhausted) {
668
+ logger.log('error', `Durable envelope ${cachedEmailArg.id}: giving up on ${retryable.length} relay recipient(s) after ${attempts} dispatch attempts: ${retryable.map((result) => `${result.recipient}=${result.message || 'failed'}`).join('; ')}`);
669
+ } else if (retryable.length > 0) {
670
+ logger.log('warn', `Durable envelope ${cachedEmailArg.id}: ${retryable.length} relay recipient(s) failed dispatch (attempt ${attempts}/${ENVELOPE_DISPATCH_MAX_ATTEMPTS}), scheduled for redispatch: ${retryable.map((result) => `${result.recipient}=${result.message || 'failed'}`).join('; ')}`);
671
+ }
672
+ }
673
+
674
+ /**
675
+ * Persist a dispatch attempt's outcome on the durable-envelope row.
676
+ *
677
+ * The row's status tracks the stored envelope, not the relay: it goes
678
+ * `deferred` only while relay recipients still need a redispatch, and returns
679
+ * to its acceptance status once none do. Relay progress itself lives in
680
+ * `routeData.envelopeDispatch`, so a relay failure can never mark a row whose
681
+ * local copy stored successfully as failed.
682
+ */
683
+ private async persistEnvelopeDispatchOutcome(
684
+ cachedEmailIdArg: string,
685
+ outcomeArg: {
686
+ attempts: number;
687
+ results: Array<{ recipient: string; status: string; message?: string; smtpCode?: number }>;
688
+ pendingRecipients: string[];
689
+ abandoned: boolean;
690
+ },
691
+ ): Promise<void> {
692
+ await this.runCachedEmailUpdate(cachedEmailIdArg, async () => {
693
+ const cachedEmail = await CachedEmail.findById(cachedEmailIdArg);
694
+ if (!cachedEmail) return;
695
+ const routeData = this.parseCachedEmailRouteData(cachedEmail);
696
+ if (!routeData.envelopeDispatch) return;
697
+ routeData.envelopeDispatch = {
698
+ ...routeData.envelopeDispatch,
699
+ attempts: outcomeArg.attempts,
700
+ results: outcomeArg.results,
701
+ pendingRecipients: outcomeArg.pendingRecipients,
702
+ ...(outcomeArg.abandoned ? { abandonedAt: new Date().toISOString() } : {}),
703
+ };
704
+ cachedEmail.routeData = JSON.stringify(routeData);
705
+
706
+ const stillPending = outcomeArg.pendingRecipients.length > 0;
707
+ const acceptanceStatus = cachedEmail.lastError && !stillPending ? 'flagged' : 'accepted';
708
+ if (stillPending) {
709
+ cachedEmail.status = 'deferred';
710
+ cachedEmail.nextAttempt = new Date(Date.now() + ENVELOPE_DISPATCH_RETRY_DELAY_MS);
711
+ } else if (cachedEmail.status === 'deferred') {
712
+ cachedEmail.status = acceptanceStatus;
713
+ cachedEmail.nextAttempt = new Date();
483
714
  }
715
+ await cachedEmail.save();
716
+ await this.notifyEmailQueuePersisted(
717
+ cachedEmail,
718
+ stillPending ? 'envelope-dispatch-deferred' : 'envelope-dispatch-settled',
719
+ );
720
+ });
721
+ }
722
+
723
+ /**
724
+ * Take exclusive ownership of a durably accepted envelope row in the spool.
725
+ *
726
+ * Runs before live-queue postponement and before the normal spool handoff: a
727
+ * relay sibling that did enqueue would otherwise postpone this row forever,
728
+ * and the normal handoff would re-run route evaluation and duplicate a
729
+ * delivery the stored copy already fulfilled.
730
+ */
731
+ private async handleDurableEnvelopeRow(
732
+ cachedEmailArg: CachedEmail,
733
+ emailServerArg: UnifiedEmailServer,
734
+ ): Promise<boolean> {
735
+ if (!this.isDurableEnvelopeRow(cachedEmailArg)) return false;
736
+ const envelopeDispatch = this.parseCachedEmailRouteData(cachedEmailArg).envelopeDispatch;
737
+
738
+ if (envelopeDispatch?.pendingRecipients?.length && !envelopeDispatch.abandonedAt) {
739
+ const rawMessage = await this.readRawMessage(cachedEmailArg);
740
+ await this.dispatchEnvelopeRecipients(
741
+ cachedEmailArg,
742
+ rawMessage.toString('utf8'),
743
+ emailServerArg,
744
+ );
745
+ return true;
484
746
  }
485
- this.trackAcceptedInboundEmail(cachedEmail);
747
+
748
+ // Nothing left to dispatch: settle the row back onto its acceptance status
749
+ // so a fully dispatched envelope does not linger as deferred.
750
+ if (cachedEmailArg.status === 'deferred') {
751
+ cachedEmailArg.status = cachedEmailArg.lastError ? 'flagged' : 'accepted';
752
+ cachedEmailArg.nextAttempt = new Date();
753
+ await cachedEmailArg.save();
754
+ await this.notifyEmailQueuePersisted(cachedEmailArg, 'envelope-dispatch-settled');
755
+ }
756
+ return true;
486
757
  }
487
758
 
488
759
  private trackAcceptedInboundEmail(cachedEmailArg: CachedEmail): void {
@@ -780,6 +1051,11 @@ export class AcceptedEmailSpool {
780
1051
  break;
781
1052
  }
782
1053
  try {
1054
+ // Durable envelopes are owned by the accepted-envelope dispatch path,
1055
+ // never by the route-evaluating handoff below.
1056
+ if (await this.handleDurableEnvelopeRow(cachedEmail, emailServer)) {
1057
+ continue;
1058
+ }
783
1059
  if (await this.postponeLiveSmartMtaOwnedEmail(cachedEmail, emailServer)) {
784
1060
  continue;
785
1061
  }
@@ -954,6 +1230,15 @@ export class AcceptedEmailSpool {
954
1230
  this.appendSmtpTransaction(cachedEmail, transaction);
955
1231
  }
956
1232
  this.updateSmartMtaRouteData(cachedEmail, item, status);
1233
+ if (this.isDurableEnvelopeRow(cachedEmail)) {
1234
+ // The row represents the durably accepted and stored envelope; the queue
1235
+ // item only covers its relay recipients. Record the relay telemetry but
1236
+ // never let a relay outcome overwrite the stored copy's status — a failed
1237
+ // relay must not mark a successfully stored envelope as failed.
1238
+ await cachedEmail.save();
1239
+ await this.notifyEmailQueuePersisted(cachedEmail, `envelope-relay-${status}`);
1240
+ return;
1241
+ }
957
1242
  if (status === 'delivered') {
958
1243
  cachedEmail.markDelivered();
959
1244
  } else if (status === 'failed') {
@@ -1016,6 +1301,11 @@ export class AcceptedEmailSpool {
1016
1301
  return cachedEmail.status === 'delivered' || cachedEmail.status === 'failed';
1017
1302
  }
1018
1303
 
1304
+ /** Whether this row was durably accepted through the envelope acceptance path. */
1305
+ private isDurableEnvelopeRow(cachedEmail: CachedEmail): boolean {
1306
+ return this.parseCachedEmailRouteData(cachedEmail).acceptance === 'durable-envelope';
1307
+ }
1308
+
1019
1309
  /** The CachedEmail row a live queue item was spooled from (via the cache-id header). */
1020
1310
  public getCachedEmailIdFromQueueItem(item: TSmartMtaQueueItemLike): string | undefined {
1021
1311
  return this.getHeaderValue(item.processingResult?.headers, DCROUTER_CACHE_ID_HEADER)