@serve.zone/dcrouter 18.1.1 → 18.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deno.json +1 -1
- package/dist_serve/bundle.js +2065 -1617
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.dcrouter.d.ts +60 -1
- package/dist_ts/classes.dcrouter.js +158 -1
- package/dist_ts/email/classes.accepted-email-spool.d.ts +66 -0
- package/dist_ts/email/classes.accepted-email-spool.js +238 -18
- package/dist_ts/email/classes.workapp-mail-manager.js +5 -2
- package/dist_ts_web/00_commitinfo_data.js +1 -1
- package/package.json +3 -3
- package/readme.md +16 -0
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.dcrouter.ts +199 -1
- package/ts/email/classes.accepted-email-spool.ts +312 -22
- package/ts/email/classes.workapp-mail-manager.ts +4 -1
- package/ts_web/00_commitinfo_data.ts +1 -1
package/ts/classes.dcrouter.ts
CHANGED
|
@@ -49,6 +49,32 @@ import type { IEmailOutboundEgressStatus, IEmailPortConfig, IEmailServerSettings
|
|
|
49
49
|
import type { IDcRouterRouteConfig, IRemoteIngressHubSettings, IRemoteIngressPerformanceConfig, TRemoteIngressHubSettingsUpdate } from '../ts_interfaces/data/remoteingress.js';
|
|
50
50
|
import type { ISecurityCompiledPolicy } from '../ts_interfaces/data/security-policy.js';
|
|
51
51
|
|
|
52
|
+
/**
|
|
53
|
+
* dcrouter's superset of SmartMTA's SMTP TLS options.
|
|
54
|
+
*
|
|
55
|
+
* SmartMTA consumes PEM content (`certPem`/`keyPem`). Deployments configure
|
|
56
|
+
* certificate FILES, so dcrouter accepts the path form too and loads it eagerly
|
|
57
|
+
* at startup — a path that cannot be read fails startup rather than silently
|
|
58
|
+
* leaving the listener without TLS.
|
|
59
|
+
*/
|
|
60
|
+
export interface IDcRouterEmailTlsConfig extends NonNullable<IUnifiedEmailServerOptions['tls']> {
|
|
61
|
+
/** Path to the certificate chain PEM file. Requires keyPath. */
|
|
62
|
+
certPath?: string;
|
|
63
|
+
/** Path to the private key PEM file. Requires certPath. */
|
|
64
|
+
keyPath?: string;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** dcrouter's email server options: SmartMTA's, with the path-shaped tls block. */
|
|
68
|
+
export type IDcRouterEmailConfig = Omit<IUnifiedEmailServerOptions, 'tls'> & {
|
|
69
|
+
tls?: IDcRouterEmailTlsConfig;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/** Whether an email configuration carries any SMTP AUTH credentials. */
|
|
73
|
+
const emailConfigHasAuth = (emailConfigArg: IUnifiedEmailServerOptions | undefined): boolean =>
|
|
74
|
+
!!emailConfigArg?.auth?.required
|
|
75
|
+
|| !!emailConfigArg?.auth?.users?.length
|
|
76
|
+
|| !!emailConfigArg?.auth?.accounts?.length;
|
|
77
|
+
|
|
52
78
|
export interface IDcRouterOptions {
|
|
53
79
|
/** Base directory for all dcrouter data. Defaults to ~/.serve.zone/dcrouter */
|
|
54
80
|
baseDir?: string;
|
|
@@ -66,7 +92,7 @@ export interface IDcRouterOptions {
|
|
|
66
92
|
* Email server configuration
|
|
67
93
|
* This enables all email handling with pattern-based routing
|
|
68
94
|
*/
|
|
69
|
-
emailConfig?:
|
|
95
|
+
emailConfig?: IDcRouterEmailConfig;
|
|
70
96
|
|
|
71
97
|
/** SmartBucket configuration for durable SmartMTA queue and attachment blobs. */
|
|
72
98
|
emailBlobStorage?: ISmartMtaBlobStorageConfig;
|
|
@@ -1623,6 +1649,10 @@ export class DcRouter {
|
|
|
1623
1649
|
expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
|
|
1624
1650
|
source: event.source,
|
|
1625
1651
|
});
|
|
1652
|
+
// Renewals arrive here too. The SMTP listener holds its PEM material as
|
|
1653
|
+
// listener-level Rust configuration, so a renewed mail-hostname
|
|
1654
|
+
// certificate must be pushed or STARTTLS keeps serving the stale one.
|
|
1655
|
+
void this.reapplyEmailTlsMaterial(event.domain);
|
|
1626
1656
|
});
|
|
1627
1657
|
|
|
1628
1658
|
// Note: smartproxy v27.5.0 emits only 'certificate-issued' and 'certificate-failed'.
|
|
@@ -1955,6 +1985,10 @@ export class DcRouter {
|
|
|
1955
1985
|
storageManager: this.smartMtaBlobStorageManager,
|
|
1956
1986
|
};
|
|
1957
1987
|
|
|
1988
|
+
const emailTls = await this.resolveEmailTlsMaterial();
|
|
1989
|
+
const tlsTerminatedPorts = this.resolveEdgeTerminatedEmailPorts(portMapping);
|
|
1990
|
+
this.logEmailAuthTransportPosture(emailConfigHasAuth(baseEmailConfig), emailTls, mappedEmailPorts, tlsTerminatedPorts, mappedSecurePort);
|
|
1991
|
+
|
|
1958
1992
|
let emailConfig: IUnifiedEmailServerOptions = await this.smtpAccountManager.composeEmailConfig({
|
|
1959
1993
|
...this.options.emailConfig,
|
|
1960
1994
|
ports: mappedEmailPorts,
|
|
@@ -1962,6 +1996,7 @@ export class DcRouter {
|
|
|
1962
1996
|
dkimKeyProvisioning: 'caller-managed',
|
|
1963
1997
|
persistRoutes: this.options.emailConfig.persistRoutes ?? false,
|
|
1964
1998
|
queue: queueOptions,
|
|
1999
|
+
...(emailTls ? { tls: { ...baseEmailConfig.tls, certPem: emailTls.certPem, keyPem: emailTls.keyPem } } : {}),
|
|
1965
2000
|
outbound: {
|
|
1966
2001
|
...baseEmailConfig.outbound,
|
|
1967
2002
|
connectionProxyProvider: (context) => this.mailEgressCoordinator.provideConnectionProxy(context),
|
|
@@ -1971,6 +2006,7 @@ export class DcRouter {
|
|
|
1971
2006
|
...(mappedSecurePort !== undefined
|
|
1972
2007
|
? { securePort: mappedSecurePort }
|
|
1973
2008
|
: {}),
|
|
2009
|
+
...(tlsTerminatedPorts.length > 0 ? { tlsTerminatedPorts } : {}),
|
|
1974
2010
|
recipientValidation: true,
|
|
1975
2011
|
proxyProtocol: {
|
|
1976
2012
|
...baseEmailConfig.smtp?.proxyProtocol,
|
|
@@ -2178,6 +2214,168 @@ export class DcRouter {
|
|
|
2178
2214
|
this.mailDnsSync.requestSync('email server start');
|
|
2179
2215
|
}
|
|
2180
2216
|
|
|
2217
|
+
/**
|
|
2218
|
+
* Resolve the PEM material for the SMTP listener.
|
|
2219
|
+
*
|
|
2220
|
+
* SmartMTA consumes `tls.certPem`/`tls.keyPem`; a path-shaped `tls` block is
|
|
2221
|
+
* ignored, which leaves the Rust listener with no TLS material at all — no
|
|
2222
|
+
* STARTTLS on the plain submission ports. Resolution order matches the
|
|
2223
|
+
* RemoteIngress tunnel precedent: explicit paths, then the ACME cert store.
|
|
2224
|
+
*
|
|
2225
|
+
* Explicitly configured paths that cannot be read FAIL STARTUP. Silently
|
|
2226
|
+
* continuing without TLS is what produced a cleartext submission port in
|
|
2227
|
+
* production, so a broken explicit configuration must be impossible to miss.
|
|
2228
|
+
*/
|
|
2229
|
+
private async resolveEmailTlsMaterial(): Promise<{ certPem: string; keyPem: string; source: string } | undefined> {
|
|
2230
|
+
const tlsConfig = this.options.emailConfig?.tls as (IUnifiedEmailServerOptions['tls'] & {
|
|
2231
|
+
certPath?: string;
|
|
2232
|
+
keyPath?: string;
|
|
2233
|
+
}) | undefined;
|
|
2234
|
+
|
|
2235
|
+
if (tlsConfig?.certPem && tlsConfig?.keyPem) {
|
|
2236
|
+
return { certPem: tlsConfig.certPem, keyPem: tlsConfig.keyPem, source: 'inline PEM' };
|
|
2237
|
+
}
|
|
2238
|
+
|
|
2239
|
+
if (tlsConfig?.certPath || tlsConfig?.keyPath) {
|
|
2240
|
+
if (!tlsConfig.certPath || !tlsConfig.keyPath) {
|
|
2241
|
+
throw new Error(
|
|
2242
|
+
'emailConfig.tls requires both certPath and keyPath when either is configured',
|
|
2243
|
+
);
|
|
2244
|
+
}
|
|
2245
|
+
let certPem: string;
|
|
2246
|
+
let keyPem: string;
|
|
2247
|
+
try {
|
|
2248
|
+
certPem = await plugins.fs.promises.readFile(tlsConfig.certPath, 'utf8');
|
|
2249
|
+
keyPem = await plugins.fs.promises.readFile(tlsConfig.keyPath, 'utf8');
|
|
2250
|
+
} catch (error: unknown) {
|
|
2251
|
+
throw new Error(
|
|
2252
|
+
`Unable to read the configured SMTP TLS material (certPath=${tlsConfig.certPath}, keyPath=${tlsConfig.keyPath}): ${(error as Error).message}`,
|
|
2253
|
+
);
|
|
2254
|
+
}
|
|
2255
|
+
if (!certPem.trim() || !keyPem.trim()) {
|
|
2256
|
+
throw new Error(
|
|
2257
|
+
`The configured SMTP TLS material is empty (certPath=${tlsConfig.certPath}, keyPath=${tlsConfig.keyPath})`,
|
|
2258
|
+
);
|
|
2259
|
+
}
|
|
2260
|
+
logger.log('info', `SMTP TLS material loaded from configured paths (${tlsConfig.certPath})`);
|
|
2261
|
+
return { certPem, keyPem, source: `configured paths (${tlsConfig.certPath})` };
|
|
2262
|
+
}
|
|
2263
|
+
|
|
2264
|
+
const mailHostname = this.options.emailConfig?.hostname;
|
|
2265
|
+
if (mailHostname) {
|
|
2266
|
+
try {
|
|
2267
|
+
const stored = await ProxyCertDoc.findByDomain(mailHostname);
|
|
2268
|
+
if (stored?.publicKey && stored?.privateKey) {
|
|
2269
|
+
logger.log('info', `SMTP TLS material loaded from the stored ACME certificate for ${mailHostname}`);
|
|
2270
|
+
return { certPem: stored.publicKey, keyPem: stored.privateKey, source: `stored ACME certificate for ${mailHostname}` };
|
|
2271
|
+
}
|
|
2272
|
+
} catch (error: unknown) {
|
|
2273
|
+
logger.log('warn', `Unable to read the stored certificate for the mail hostname ${mailHostname}: ${(error as Error).message}`);
|
|
2274
|
+
}
|
|
2275
|
+
}
|
|
2276
|
+
|
|
2277
|
+
return undefined;
|
|
2278
|
+
}
|
|
2279
|
+
|
|
2280
|
+
/**
|
|
2281
|
+
* Push renewed TLS material to the running SMTP listener.
|
|
2282
|
+
*
|
|
2283
|
+
* Only reacts to the mail hostname, and only when explicit paths are NOT
|
|
2284
|
+
* configured — an operator-managed file pair is not superseded by an ACME
|
|
2285
|
+
* renewal for the same name. SmartMTA restarts just the Rust listener when
|
|
2286
|
+
* the PEM material actually changes.
|
|
2287
|
+
*/
|
|
2288
|
+
private async reapplyEmailTlsMaterial(domainArg: string): Promise<void> {
|
|
2289
|
+
const emailServer = this.emailServer;
|
|
2290
|
+
const mailHostname = this.options.emailConfig?.hostname;
|
|
2291
|
+
if (!emailServer || !mailHostname) return;
|
|
2292
|
+
if (domainArg.toLowerCase() !== mailHostname.toLowerCase()) return;
|
|
2293
|
+
const tlsConfig = this.options.emailConfig?.tls;
|
|
2294
|
+
if (tlsConfig?.certPath || tlsConfig?.keyPath) return;
|
|
2295
|
+
|
|
2296
|
+
try {
|
|
2297
|
+
const emailTls = await this.resolveEmailTlsMaterial();
|
|
2298
|
+
if (!emailTls) return;
|
|
2299
|
+
emailServer.updateOptions({
|
|
2300
|
+
tls: { ...this.options.emailConfig?.tls, certPem: emailTls.certPem, keyPem: emailTls.keyPem },
|
|
2301
|
+
});
|
|
2302
|
+
if (this.options.emailConfig) {
|
|
2303
|
+
this.options.emailConfig.tls = {
|
|
2304
|
+
...this.options.emailConfig.tls,
|
|
2305
|
+
certPem: emailTls.certPem,
|
|
2306
|
+
keyPem: emailTls.keyPem,
|
|
2307
|
+
};
|
|
2308
|
+
}
|
|
2309
|
+
logger.log('info', `Pushed renewed SMTP TLS material for ${mailHostname} to the email listener`);
|
|
2310
|
+
} catch (error: unknown) {
|
|
2311
|
+
logger.log('error', `Unable to apply renewed SMTP TLS material for ${mailHostname}: ${(error as Error).message}`);
|
|
2312
|
+
}
|
|
2313
|
+
}
|
|
2314
|
+
|
|
2315
|
+
/**
|
|
2316
|
+
* Internal SMTP ports whose public leg is TLS-terminated by CoreTraffic.
|
|
2317
|
+
*
|
|
2318
|
+
* Those backend legs arrive as plaintext even though the client's channel was
|
|
2319
|
+
* encrypted, so without declaring them the AUTH-requires-encryption gate would
|
|
2320
|
+
* refuse authentication on the only encrypted submission port. Derived from
|
|
2321
|
+
* the generated route set rather than hardcoding 465, so it stays true when
|
|
2322
|
+
* emailPortConfig changes.
|
|
2323
|
+
*/
|
|
2324
|
+
private resolveEdgeTerminatedEmailPorts(portMapping: Record<number, number>): number[] {
|
|
2325
|
+
if (!this.options.emailConfig) return [];
|
|
2326
|
+
const terminatedPorts: number[] = [];
|
|
2327
|
+
for (const route of this.emailRouteBuilder.generateEmailRoutes(this.options.emailConfig)) {
|
|
2328
|
+
if ((route.action as { tls?: { mode?: string } }).tls?.mode !== 'terminate') continue;
|
|
2329
|
+
const publicPorts = Array.isArray(route.match?.ports) ? route.match.ports : [];
|
|
2330
|
+
for (const publicPort of publicPorts) {
|
|
2331
|
+
if (typeof publicPort !== 'number') continue;
|
|
2332
|
+
const internalPort = portMapping[publicPort] || publicPort + 10000;
|
|
2333
|
+
// The implicit-TLS securePort is terminated by smartmta itself.
|
|
2334
|
+
if (internalPort === this.options.emailConfig.smtp?.securePort) continue;
|
|
2335
|
+
if (!terminatedPorts.includes(internalPort)) {
|
|
2336
|
+
terminatedPorts.push(internalPort);
|
|
2337
|
+
}
|
|
2338
|
+
}
|
|
2339
|
+
}
|
|
2340
|
+
return terminatedPorts;
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
/**
|
|
2344
|
+
* State the transport posture of every SMTP listener port at startup.
|
|
2345
|
+
*
|
|
2346
|
+
* AUTH is only offered on an encrypted transport, so an operator has to be
|
|
2347
|
+
* able to see at a glance which submission ports can actually authenticate —
|
|
2348
|
+
* missing certificate material silently disabling AUTH on 587 would otherwise
|
|
2349
|
+
* look like a client problem.
|
|
2350
|
+
*/
|
|
2351
|
+
private logEmailAuthTransportPosture(
|
|
2352
|
+
authConfigured: boolean,
|
|
2353
|
+
emailTls: { source: string } | undefined,
|
|
2354
|
+
mappedPorts: number[],
|
|
2355
|
+
tlsTerminatedPorts: number[],
|
|
2356
|
+
mappedSecurePort: number | undefined,
|
|
2357
|
+
): void {
|
|
2358
|
+
const authCapable: number[] = [];
|
|
2359
|
+
const cleartext: number[] = [];
|
|
2360
|
+
for (const port of mappedPorts) {
|
|
2361
|
+
if (port === mappedSecurePort || tlsTerminatedPorts.includes(port) || emailTls) {
|
|
2362
|
+
authCapable.push(port);
|
|
2363
|
+
} else {
|
|
2364
|
+
cleartext.push(port);
|
|
2365
|
+
}
|
|
2366
|
+
}
|
|
2367
|
+
logger.log(
|
|
2368
|
+
'info',
|
|
2369
|
+
`SMTP transport posture: TLS material=${emailTls ? emailTls.source : 'NONE'}, edge-terminated ports=[${tlsTerminatedPorts.join(', ') || 'none'}], implicit-TLS port=${mappedSecurePort ?? 'none'}, AUTH-capable ports=[${authCapable.join(', ') || 'none'}]`,
|
|
2370
|
+
);
|
|
2371
|
+
if (authConfigured && !emailTls && cleartext.length > 0) {
|
|
2372
|
+
logger.log(
|
|
2373
|
+
'error',
|
|
2374
|
+
`SMTP AUTH is configured but ports [${cleartext.join(', ')}] have no TLS material and no upstream TLS terminator — AUTH is refused there because credentials must never cross a cleartext channel. Configure emailConfig.tls.certPath/keyPath, or provision an ACME certificate for ${this.options.emailConfig?.hostname || 'the mail hostname'}.`,
|
|
2375
|
+
);
|
|
2376
|
+
}
|
|
2377
|
+
}
|
|
2378
|
+
|
|
2181
2379
|
/**
|
|
2182
2380
|
* Readiness of the RemoteIngress outbound mail egress path (mail-tagged edges).
|
|
2183
2381
|
*/
|
|
@@ -8,10 +8,13 @@ import type {
|
|
|
8
8
|
import { AcceptEnvelopeRejectionError } from '@push.rocks/smartmta';
|
|
9
9
|
import type {
|
|
10
10
|
Email,
|
|
11
|
+
IAcceptedEnvelopeDispatchMetadata,
|
|
12
|
+
IAcceptedEnvelopeRecipientPlan,
|
|
11
13
|
IAcceptEnvelopeContext,
|
|
12
14
|
IExtendedSmtpSession,
|
|
13
15
|
IMessageAcceptanceContext,
|
|
14
16
|
IMessageAcceptanceDecision,
|
|
17
|
+
IResolvedRecipientRoute,
|
|
15
18
|
UnifiedEmailServer,
|
|
16
19
|
} from '@push.rocks/smartmta';
|
|
17
20
|
import type { DcRouter } from '../classes.dcrouter.js';
|
|
@@ -76,6 +79,9 @@ const ACCEPTED_EMAIL_SPOOL_BATCH_SIZE = 25;
|
|
|
76
79
|
const ACCEPTED_EMAIL_STOP_DRAIN_TIMEOUT_MS = 30_000;
|
|
77
80
|
/** Retention for catch-all stored inbound mail (30 days). */
|
|
78
81
|
const INBOUND_STORE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
|
|
82
|
+
/** Redispatch attempts for the non-store recipients of a durable envelope. */
|
|
83
|
+
const ENVELOPE_DISPATCH_MAX_ATTEMPTS = 10;
|
|
84
|
+
const ENVELOPE_DISPATCH_RETRY_DELAY_MS = 5 * 60_000;
|
|
79
85
|
|
|
80
86
|
/**
|
|
81
87
|
* Permanent per-email storage failure: the raw RFC822 payload of an accepted
|
|
@@ -136,8 +142,32 @@ type TStoredCachedEmailSession = {
|
|
|
136
142
|
};
|
|
137
143
|
};
|
|
138
144
|
|
|
145
|
+
/**
|
|
146
|
+
* Persisted per-recipient dispatch state for a durably accepted envelope.
|
|
147
|
+
* Holds everything needed to replay `dispatchAcceptedEnvelope` byte-identically:
|
|
148
|
+
* upstream fingerprints each recipient over the raw message, the metadata and
|
|
149
|
+
* the plan entry, so a replay reconstructed from anything else would be refused
|
|
150
|
+
* as an idempotency-key reuse instead of retrying.
|
|
151
|
+
*/
|
|
152
|
+
type TStoredEnvelopeDispatch = {
|
|
153
|
+
plan: IAcceptedEnvelopeRecipientPlan[];
|
|
154
|
+
metadata: IAcceptedEnvelopeDispatchMetadata;
|
|
155
|
+
attempts: number;
|
|
156
|
+
results?: Array<{
|
|
157
|
+
recipient: string;
|
|
158
|
+
status: string;
|
|
159
|
+
message?: string;
|
|
160
|
+
smtpCode?: number;
|
|
161
|
+
}>;
|
|
162
|
+
pendingRecipients?: string[];
|
|
163
|
+
/** Set once retries are exhausted; the row stops being redispatched. */
|
|
164
|
+
abandonedAt?: string;
|
|
165
|
+
};
|
|
166
|
+
|
|
139
167
|
type TStoredCachedEmailRouteData = {
|
|
168
|
+
acceptance?: string;
|
|
140
169
|
session?: TStoredCachedEmailSession;
|
|
170
|
+
envelopeDispatch?: TStoredEnvelopeDispatch;
|
|
141
171
|
smartMta?: {
|
|
142
172
|
status?: 'queued' | 'deferred' | 'delivered' | 'failed';
|
|
143
173
|
nextAttempt?: string;
|
|
@@ -185,6 +215,54 @@ export class AcceptedEmailSpool {
|
|
|
185
215
|
|
|
186
216
|
constructor(private dcRouterRef: DcRouter) {}
|
|
187
217
|
|
|
218
|
+
/**
|
|
219
|
+
* Direction is decided by the RECIPIENT, never by whether the session
|
|
220
|
+
* authenticated. Authentication grants permission to relay; it does not make
|
|
221
|
+
* a message addressed to a mailbox we host into outbound mail. A message with
|
|
222
|
+
* at least one locally hosted recipient is inbound — a local mailbox receives
|
|
223
|
+
* it — and only an envelope addressed exclusively to remote recipients is
|
|
224
|
+
* outbound.
|
|
225
|
+
*
|
|
226
|
+
* `IResolvedRecipientRoute.localDomain` is smartmta's own per-recipient
|
|
227
|
+
* verdict (non-null exactly when the domain registry hosts the domain), so
|
|
228
|
+
* the classification uses the same truth the routing decision used. When an
|
|
229
|
+
* acceptance context carries no resolution the envelope recipients are
|
|
230
|
+
* classified against the live registry instead — never against the session,
|
|
231
|
+
* which is the mistake being fixed.
|
|
232
|
+
*/
|
|
233
|
+
private deriveDirectionFromResolvedRoutes(
|
|
234
|
+
resolvedRecipientRoutes: readonly IResolvedRecipientRoute[] | undefined,
|
|
235
|
+
envelopeRecipientsArg: readonly string[],
|
|
236
|
+
): TCachedEmailDirection {
|
|
237
|
+
if (resolvedRecipientRoutes?.length) {
|
|
238
|
+
return resolvedRecipientRoutes.some((resolution) => !!resolution.localDomain)
|
|
239
|
+
? 'inbound'
|
|
240
|
+
: 'outbound';
|
|
241
|
+
}
|
|
242
|
+
return this.deriveDirectionFromRecipients(envelopeRecipientsArg);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Recipient-derived direction for programmatic submitters, which have no SMTP
|
|
247
|
+
* recipient resolution to consult. Falls back to the live domain registry.
|
|
248
|
+
*/
|
|
249
|
+
private deriveDirectionFromRecipients(recipientsArg: readonly string[]): TCachedEmailDirection {
|
|
250
|
+
const domainRegistry = this.dcRouterRef.emailServer?.domainRegistry;
|
|
251
|
+
if (!domainRegistry) {
|
|
252
|
+
// Without the registry there is no recipient truth to classify against.
|
|
253
|
+
// Programmatic submission is relay by construction, so outbound is the
|
|
254
|
+
// honest label rather than guessing from the session.
|
|
255
|
+
return 'outbound';
|
|
256
|
+
}
|
|
257
|
+
for (const recipient of recipientsArg) {
|
|
258
|
+
const domain = recipient.split('@')[1]?.trim().toLowerCase();
|
|
259
|
+
if (domain && domainRegistry.isDomainRegistered(domain)) {
|
|
260
|
+
return 'inbound';
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return 'outbound';
|
|
264
|
+
}
|
|
265
|
+
|
|
188
266
|
public async acceptMessage(
|
|
189
267
|
context: IMessageAcceptanceContext,
|
|
190
268
|
processAfterAccept = true,
|
|
@@ -218,7 +296,10 @@ export class AcceptedEmailSpool {
|
|
|
218
296
|
await this.persistRawMessage(cachedEmail, persistedRawMessage);
|
|
219
297
|
cachedEmail.status = 'pending';
|
|
220
298
|
cachedEmail.nextAttempt = new Date();
|
|
221
|
-
cachedEmail.direction =
|
|
299
|
+
cachedEmail.direction = this.deriveDirectionFromResolvedRoutes(
|
|
300
|
+
context.resolvedRecipientRoutes,
|
|
301
|
+
envelopeRecipients.length > 0 ? envelopeRecipients : cachedEmail.to,
|
|
302
|
+
);
|
|
222
303
|
cachedEmail.acceptedAt = Date.now();
|
|
223
304
|
cachedEmail.routeData = JSON.stringify({
|
|
224
305
|
acceptedAt: new Date().toISOString(),
|
|
@@ -288,7 +369,7 @@ export class AcceptedEmailSpool {
|
|
|
288
369
|
cachedEmail.status = 'pending';
|
|
289
370
|
cachedEmail.nextAttempt = new Date();
|
|
290
371
|
cachedEmail.direction = optionsArg.direction
|
|
291
|
-
?? (optionsArg.
|
|
372
|
+
?? this.deriveDirectionFromRecipients(optionsArg.envelope.rcptTo);
|
|
292
373
|
cachedEmail.acceptedAt = Date.now();
|
|
293
374
|
cachedEmail.routeData = JSON.stringify({
|
|
294
375
|
acceptedAt: new Date().toISOString(),
|
|
@@ -379,7 +460,10 @@ export class AcceptedEmailSpool {
|
|
|
379
460
|
cachedEmail.id,
|
|
380
461
|
);
|
|
381
462
|
await this.persistRawMessage(cachedEmail, persistedRawMessage);
|
|
382
|
-
cachedEmail.direction =
|
|
463
|
+
cachedEmail.direction = this.deriveDirectionFromResolvedRoutes(
|
|
464
|
+
context.resolvedRecipientRoutes,
|
|
465
|
+
context.envelope.rcptTo,
|
|
466
|
+
);
|
|
383
467
|
cachedEmail.acceptedAt = startedAtMs;
|
|
384
468
|
|
|
385
469
|
if (acceptance.dmarcReject) {
|
|
@@ -415,6 +499,15 @@ export class AcceptedEmailSpool {
|
|
|
415
499
|
} else {
|
|
416
500
|
cachedEmail.status = 'accepted';
|
|
417
501
|
}
|
|
502
|
+
if (nonStoreEntries.length > 0) {
|
|
503
|
+
// Relay recipients are dispatched after this row is committed. Park the
|
|
504
|
+
// row in a status the spool actually scans so a crash between the commit
|
|
505
|
+
// and the dispatch leaves those recipients recoverable instead of stranded
|
|
506
|
+
// in a terminal-looking 'accepted' row. The dispatch outcome settles it
|
|
507
|
+
// straight back to the acceptance status.
|
|
508
|
+
cachedEmail.status = 'deferred';
|
|
509
|
+
cachedEmail.nextAttempt = new Date(Date.now() + ENVELOPE_DISPATCH_RETRY_DELAY_MS);
|
|
510
|
+
}
|
|
418
511
|
cachedEmail.deliveredAt = new Date();
|
|
419
512
|
cachedEmail.setTTL(INBOUND_STORE_RETENTION_MS);
|
|
420
513
|
cachedEmail.appendSmtpTransaction(this.buildInboundTransaction(context, cachedEmail.id, {
|
|
@@ -424,6 +517,7 @@ export class AcceptedEmailSpool {
|
|
|
424
517
|
doubts: acceptance.doubts,
|
|
425
518
|
startedAtMs,
|
|
426
519
|
}));
|
|
520
|
+
const dispatchMetadata = this.buildEnvelopeDispatchMetadata(context);
|
|
427
521
|
cachedEmail.routeData = JSON.stringify({
|
|
428
522
|
acceptedAt: new Date(startedAtMs).toISOString(),
|
|
429
523
|
acceptance: 'durable-envelope',
|
|
@@ -434,6 +528,18 @@ export class AcceptedEmailSpool {
|
|
|
434
528
|
source: entry.source,
|
|
435
529
|
actionType: entry.action.type,
|
|
436
530
|
})),
|
|
531
|
+
// Persisted BEFORE the SMTP 250, so a dispatch failure for the non-store
|
|
532
|
+
// recipients is recoverable instead of being lost with the process.
|
|
533
|
+
...(nonStoreEntries.length > 0
|
|
534
|
+
? {
|
|
535
|
+
envelopeDispatch: {
|
|
536
|
+
plan: [...plan],
|
|
537
|
+
metadata: dispatchMetadata,
|
|
538
|
+
attempts: 0,
|
|
539
|
+
pendingRecipients: nonStoreEntries.map((entry) => entry.recipient),
|
|
540
|
+
} satisfies TStoredEnvelopeDispatch,
|
|
541
|
+
}
|
|
542
|
+
: {}),
|
|
437
543
|
session: {
|
|
438
544
|
id: session.id,
|
|
439
545
|
remoteAddress: session.remoteAddress,
|
|
@@ -461,28 +567,193 @@ export class AcceptedEmailSpool {
|
|
|
461
567
|
}
|
|
462
568
|
|
|
463
569
|
if (nonStoreEntries.length > 0) {
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
570
|
+
await this.dispatchEnvelopeRecipients(cachedEmail, persistedRawMessage, emailServer);
|
|
571
|
+
}
|
|
572
|
+
this.trackAcceptedInboundEmail(cachedEmail);
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
/**
|
|
576
|
+
* Session metadata for `dispatchAcceptedEnvelope`. It feeds the per-recipient
|
|
577
|
+
* idempotency fingerprint, so it must be JSON-round-trip stable: a replay
|
|
578
|
+
* reconstructed from persisted state has to produce byte-identical metadata
|
|
579
|
+
* or upstream refuses it as an idempotency-key reuse.
|
|
580
|
+
*/
|
|
581
|
+
private buildEnvelopeDispatchMetadata(
|
|
582
|
+
context: IAcceptEnvelopeContext,
|
|
583
|
+
): IAcceptedEnvelopeDispatchMetadata {
|
|
584
|
+
const session = context.session;
|
|
585
|
+
return {
|
|
586
|
+
mailFrom: context.envelope.mailFrom,
|
|
587
|
+
session: {
|
|
588
|
+
id: session.id || '',
|
|
589
|
+
remoteAddress: session.remoteAddress || '',
|
|
590
|
+
clientHostname: session.clientHostname || '',
|
|
591
|
+
secure: !!session.secure,
|
|
592
|
+
authenticated: !!session.authenticated,
|
|
593
|
+
},
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
/**
|
|
598
|
+
* Dispatch (or redispatch) the non-store recipients of a durably accepted
|
|
599
|
+
* envelope.
|
|
600
|
+
*
|
|
601
|
+
* The raw bytes handed to upstream are the exact bytes persisted for this row,
|
|
602
|
+
* because upstream fingerprints every recipient over the raw message: a later
|
|
603
|
+
* replay with different bytes would be rejected as an idempotency-key reuse
|
|
604
|
+
* rather than retried. Recipients that already succeeded are short-circuited
|
|
605
|
+
* by upstream's checkpoints, and `failed` results are deliberately not
|
|
606
|
+
* checkpointed upstream, so an identical replay retries exactly those.
|
|
607
|
+
*
|
|
608
|
+
* A `failed` recipient leaves the row non-terminal so the spool retries it;
|
|
609
|
+
* a `rejected` recipient is a permanent per-recipient refusal and is recorded
|
|
610
|
+
* durably instead of being retried. Either way the outcome is persisted — a
|
|
611
|
+
* relay recipient is never silently dropped after the SMTP 250.
|
|
612
|
+
*/
|
|
613
|
+
private async dispatchEnvelopeRecipients(
|
|
614
|
+
cachedEmailArg: CachedEmail,
|
|
615
|
+
rawMessageArg: string,
|
|
616
|
+
emailServerArg: UnifiedEmailServer,
|
|
617
|
+
): Promise<void> {
|
|
618
|
+
const routeData = this.parseCachedEmailRouteData(cachedEmailArg);
|
|
619
|
+
const envelopeDispatch = routeData.envelopeDispatch;
|
|
620
|
+
if (!envelopeDispatch || envelopeDispatch.abandonedAt) return;
|
|
621
|
+
|
|
622
|
+
const attempts = (envelopeDispatch.attempts || 0) + 1;
|
|
623
|
+
let dispatchResults: Array<{
|
|
624
|
+
recipient: string;
|
|
625
|
+
status: string;
|
|
626
|
+
message?: string;
|
|
627
|
+
smtpCode?: number;
|
|
628
|
+
}>;
|
|
629
|
+
try {
|
|
630
|
+
const dispatchResult = await emailServerArg.dispatchAcceptedEnvelope(
|
|
631
|
+
plugins.buffer.Buffer.from(rawMessageArg, 'utf8'),
|
|
632
|
+
envelopeDispatch.plan,
|
|
633
|
+
envelopeDispatch.metadata,
|
|
480
634
|
);
|
|
481
|
-
|
|
482
|
-
|
|
635
|
+
dispatchResults = dispatchResult.results.map((result) => ({
|
|
636
|
+
recipient: result.recipient,
|
|
637
|
+
status: result.status,
|
|
638
|
+
...(result.message ? { message: result.message } : {}),
|
|
639
|
+
...(result.smtpCode !== undefined ? { smtpCode: result.smtpCode } : {}),
|
|
640
|
+
}));
|
|
641
|
+
} catch (error: unknown) {
|
|
642
|
+
// A whole-call failure (missing managed queue storage, malformed
|
|
643
|
+
// checkpoint) is retried the same way a per-recipient failure is.
|
|
644
|
+
dispatchResults = envelopeDispatch.plan
|
|
645
|
+
.filter((entry) => entry.action.type !== 'store' && entry.action.type !== 'deliver')
|
|
646
|
+
.map((entry) => ({
|
|
647
|
+
recipient: entry.recipient,
|
|
648
|
+
status: 'failed',
|
|
649
|
+
message: (error as Error).message,
|
|
650
|
+
}));
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
const retryable = dispatchResults.filter((result) => result.status === 'failed');
|
|
654
|
+
const rejected = dispatchResults.filter((result) => result.status === 'rejected');
|
|
655
|
+
const exhausted = retryable.length > 0 && attempts >= ENVELOPE_DISPATCH_MAX_ATTEMPTS;
|
|
656
|
+
|
|
657
|
+
await this.persistEnvelopeDispatchOutcome(cachedEmailArg.id, {
|
|
658
|
+
attempts,
|
|
659
|
+
results: dispatchResults,
|
|
660
|
+
pendingRecipients: exhausted ? [] : retryable.map((result) => result.recipient),
|
|
661
|
+
abandoned: exhausted,
|
|
662
|
+
});
|
|
663
|
+
|
|
664
|
+
if (rejected.length > 0) {
|
|
665
|
+
logger.log('error', `Durable envelope ${cachedEmailArg.id}: ${rejected.length} relay recipient(s) permanently refused: ${rejected.map((result) => `${result.recipient}=${result.smtpCode || 550} ${result.message || 'rejected'}`).join('; ')}`);
|
|
666
|
+
}
|
|
667
|
+
if (exhausted) {
|
|
668
|
+
logger.log('error', `Durable envelope ${cachedEmailArg.id}: giving up on ${retryable.length} relay recipient(s) after ${attempts} dispatch attempts: ${retryable.map((result) => `${result.recipient}=${result.message || 'failed'}`).join('; ')}`);
|
|
669
|
+
} else if (retryable.length > 0) {
|
|
670
|
+
logger.log('warn', `Durable envelope ${cachedEmailArg.id}: ${retryable.length} relay recipient(s) failed dispatch (attempt ${attempts}/${ENVELOPE_DISPATCH_MAX_ATTEMPTS}), scheduled for redispatch: ${retryable.map((result) => `${result.recipient}=${result.message || 'failed'}`).join('; ')}`);
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* Persist a dispatch attempt's outcome on the durable-envelope row.
|
|
676
|
+
*
|
|
677
|
+
* The row's status tracks the stored envelope, not the relay: it goes
|
|
678
|
+
* `deferred` only while relay recipients still need a redispatch, and returns
|
|
679
|
+
* to its acceptance status once none do. Relay progress itself lives in
|
|
680
|
+
* `routeData.envelopeDispatch`, so a relay failure can never mark a row whose
|
|
681
|
+
* local copy stored successfully as failed.
|
|
682
|
+
*/
|
|
683
|
+
private async persistEnvelopeDispatchOutcome(
|
|
684
|
+
cachedEmailIdArg: string,
|
|
685
|
+
outcomeArg: {
|
|
686
|
+
attempts: number;
|
|
687
|
+
results: Array<{ recipient: string; status: string; message?: string; smtpCode?: number }>;
|
|
688
|
+
pendingRecipients: string[];
|
|
689
|
+
abandoned: boolean;
|
|
690
|
+
},
|
|
691
|
+
): Promise<void> {
|
|
692
|
+
await this.runCachedEmailUpdate(cachedEmailIdArg, async () => {
|
|
693
|
+
const cachedEmail = await CachedEmail.findById(cachedEmailIdArg);
|
|
694
|
+
if (!cachedEmail) return;
|
|
695
|
+
const routeData = this.parseCachedEmailRouteData(cachedEmail);
|
|
696
|
+
if (!routeData.envelopeDispatch) return;
|
|
697
|
+
routeData.envelopeDispatch = {
|
|
698
|
+
...routeData.envelopeDispatch,
|
|
699
|
+
attempts: outcomeArg.attempts,
|
|
700
|
+
results: outcomeArg.results,
|
|
701
|
+
pendingRecipients: outcomeArg.pendingRecipients,
|
|
702
|
+
...(outcomeArg.abandoned ? { abandonedAt: new Date().toISOString() } : {}),
|
|
703
|
+
};
|
|
704
|
+
cachedEmail.routeData = JSON.stringify(routeData);
|
|
705
|
+
|
|
706
|
+
const stillPending = outcomeArg.pendingRecipients.length > 0;
|
|
707
|
+
const acceptanceStatus = cachedEmail.lastError && !stillPending ? 'flagged' : 'accepted';
|
|
708
|
+
if (stillPending) {
|
|
709
|
+
cachedEmail.status = 'deferred';
|
|
710
|
+
cachedEmail.nextAttempt = new Date(Date.now() + ENVELOPE_DISPATCH_RETRY_DELAY_MS);
|
|
711
|
+
} else if (cachedEmail.status === 'deferred') {
|
|
712
|
+
cachedEmail.status = acceptanceStatus;
|
|
713
|
+
cachedEmail.nextAttempt = new Date();
|
|
483
714
|
}
|
|
715
|
+
await cachedEmail.save();
|
|
716
|
+
await this.notifyEmailQueuePersisted(
|
|
717
|
+
cachedEmail,
|
|
718
|
+
stillPending ? 'envelope-dispatch-deferred' : 'envelope-dispatch-settled',
|
|
719
|
+
);
|
|
720
|
+
});
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
/**
|
|
724
|
+
* Take exclusive ownership of a durably accepted envelope row in the spool.
|
|
725
|
+
*
|
|
726
|
+
* Runs before live-queue postponement and before the normal spool handoff: a
|
|
727
|
+
* relay sibling that did enqueue would otherwise postpone this row forever,
|
|
728
|
+
* and the normal handoff would re-run route evaluation and duplicate a
|
|
729
|
+
* delivery the stored copy already fulfilled.
|
|
730
|
+
*/
|
|
731
|
+
private async handleDurableEnvelopeRow(
|
|
732
|
+
cachedEmailArg: CachedEmail,
|
|
733
|
+
emailServerArg: UnifiedEmailServer,
|
|
734
|
+
): Promise<boolean> {
|
|
735
|
+
if (!this.isDurableEnvelopeRow(cachedEmailArg)) return false;
|
|
736
|
+
const envelopeDispatch = this.parseCachedEmailRouteData(cachedEmailArg).envelopeDispatch;
|
|
737
|
+
|
|
738
|
+
if (envelopeDispatch?.pendingRecipients?.length && !envelopeDispatch.abandonedAt) {
|
|
739
|
+
const rawMessage = await this.readRawMessage(cachedEmailArg);
|
|
740
|
+
await this.dispatchEnvelopeRecipients(
|
|
741
|
+
cachedEmailArg,
|
|
742
|
+
rawMessage.toString('utf8'),
|
|
743
|
+
emailServerArg,
|
|
744
|
+
);
|
|
745
|
+
return true;
|
|
484
746
|
}
|
|
485
|
-
|
|
747
|
+
|
|
748
|
+
// Nothing left to dispatch: settle the row back onto its acceptance status
|
|
749
|
+
// so a fully dispatched envelope does not linger as deferred.
|
|
750
|
+
if (cachedEmailArg.status === 'deferred') {
|
|
751
|
+
cachedEmailArg.status = cachedEmailArg.lastError ? 'flagged' : 'accepted';
|
|
752
|
+
cachedEmailArg.nextAttempt = new Date();
|
|
753
|
+
await cachedEmailArg.save();
|
|
754
|
+
await this.notifyEmailQueuePersisted(cachedEmailArg, 'envelope-dispatch-settled');
|
|
755
|
+
}
|
|
756
|
+
return true;
|
|
486
757
|
}
|
|
487
758
|
|
|
488
759
|
private trackAcceptedInboundEmail(cachedEmailArg: CachedEmail): void {
|
|
@@ -780,6 +1051,11 @@ export class AcceptedEmailSpool {
|
|
|
780
1051
|
break;
|
|
781
1052
|
}
|
|
782
1053
|
try {
|
|
1054
|
+
// Durable envelopes are owned by the accepted-envelope dispatch path,
|
|
1055
|
+
// never by the route-evaluating handoff below.
|
|
1056
|
+
if (await this.handleDurableEnvelopeRow(cachedEmail, emailServer)) {
|
|
1057
|
+
continue;
|
|
1058
|
+
}
|
|
783
1059
|
if (await this.postponeLiveSmartMtaOwnedEmail(cachedEmail, emailServer)) {
|
|
784
1060
|
continue;
|
|
785
1061
|
}
|
|
@@ -954,6 +1230,15 @@ export class AcceptedEmailSpool {
|
|
|
954
1230
|
this.appendSmtpTransaction(cachedEmail, transaction);
|
|
955
1231
|
}
|
|
956
1232
|
this.updateSmartMtaRouteData(cachedEmail, item, status);
|
|
1233
|
+
if (this.isDurableEnvelopeRow(cachedEmail)) {
|
|
1234
|
+
// The row represents the durably accepted and stored envelope; the queue
|
|
1235
|
+
// item only covers its relay recipients. Record the relay telemetry but
|
|
1236
|
+
// never let a relay outcome overwrite the stored copy's status — a failed
|
|
1237
|
+
// relay must not mark a successfully stored envelope as failed.
|
|
1238
|
+
await cachedEmail.save();
|
|
1239
|
+
await this.notifyEmailQueuePersisted(cachedEmail, `envelope-relay-${status}`);
|
|
1240
|
+
return;
|
|
1241
|
+
}
|
|
957
1242
|
if (status === 'delivered') {
|
|
958
1243
|
cachedEmail.markDelivered();
|
|
959
1244
|
} else if (status === 'failed') {
|
|
@@ -1016,6 +1301,11 @@ export class AcceptedEmailSpool {
|
|
|
1016
1301
|
return cachedEmail.status === 'delivered' || cachedEmail.status === 'failed';
|
|
1017
1302
|
}
|
|
1018
1303
|
|
|
1304
|
+
/** Whether this row was durably accepted through the envelope acceptance path. */
|
|
1305
|
+
private isDurableEnvelopeRow(cachedEmail: CachedEmail): boolean {
|
|
1306
|
+
return this.parseCachedEmailRouteData(cachedEmail).acceptance === 'durable-envelope';
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1019
1309
|
/** The CachedEmail row a live queue item was spooled from (via the cache-id header). */
|
|
1020
1310
|
public getCachedEmailIdFromQueueItem(item: TSmartMtaQueueItemLike): string | undefined {
|
|
1021
1311
|
return this.getHeaderValue(item.processingResult?.headers, DCROUTER_CACHE_ID_HEADER)
|