@serve.zone/dcrouter 18.1.0 → 18.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deno.json +1 -1
- package/dist_serve/bundle.js +2072 -1624
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.dcrouter.d.ts +64 -2
- package/dist_ts/classes.dcrouter.js +187 -6
- package/dist_ts/db/documents/classes.authentication-event.doc.d.ts +25 -0
- package/dist_ts/db/documents/classes.authentication-event.doc.js +273 -0
- package/dist_ts/db/documents/classes.email-traffic-bucket.doc.d.ts +25 -0
- package/dist_ts/db/documents/classes.email-traffic-bucket.doc.js +200 -0
- package/dist_ts/db/documents/index.d.ts +2 -0
- package/dist_ts/db/documents/index.js +3 -1
- package/dist_ts/email/classes.accepted-email-spool.d.ts +67 -0
- package/dist_ts/email/classes.accepted-email-spool.js +245 -17
- package/dist_ts/email/classes.workapp-mail-manager.js +5 -2
- package/dist_ts/monitoring/classes.metricsmanager.d.ts +19 -0
- package/dist_ts/monitoring/classes.metricsmanager.js +240 -44
- package/dist_ts/opsserver/handlers/admin.handler.d.ts +1 -0
- package/dist_ts/opsserver/handlers/admin.handler.js +61 -14
- package/dist_ts/opsserver/handlers/security.handler.js +21 -3
- package/dist_ts/opsserver/handlers/stats.handler.js +27 -10
- package/dist_ts/security/classes.authentication-event-manager.d.ts +35 -0
- package/dist_ts/security/classes.authentication-event-manager.js +185 -0
- package/dist_ts/security/index.d.ts +1 -0
- package/dist_ts/security/index.js +2 -1
- package/dist_ts_interfaces/data/stats.d.ts +16 -0
- package/dist_ts_migrations/index.js +74 -15
- package/dist_ts_web/00_commitinfo_data.js +1 -1
- package/dist_ts_web/elements/overview/ops-view-overview.js +3 -2
- package/dist_ts_web/elements/security/ops-view-security-authentication.js +20 -13
- package/dist_ts_web/elements/security/ops-view-security-overview.js +8 -11
- package/package.json +3 -3
- package/readme.md +24 -0
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.dcrouter.ts +230 -6
- package/ts/db/documents/classes.authentication-event.doc.ts +248 -0
- package/ts/db/documents/classes.email-traffic-bucket.doc.ts +161 -0
- package/ts/db/documents/index.ts +2 -0
- package/ts/email/classes.accepted-email-spool.ts +319 -21
- package/ts/email/classes.workapp-mail-manager.ts +4 -1
- package/ts/monitoring/classes.metricsmanager.ts +287 -50
- package/ts/opsserver/handlers/admin.handler.ts +74 -15
- package/ts/opsserver/handlers/security.handler.ts +23 -2
- package/ts/opsserver/handlers/stats.handler.ts +29 -9
- package/ts/security/classes.authentication-event-manager.ts +221 -0
- package/ts/security/index.ts +1 -0
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/elements/overview/ops-view-overview.ts +2 -1
- package/ts_web/elements/security/ops-view-security-authentication.ts +20 -13
- package/ts_web/elements/security/ops-view-security-overview.ts +7 -12
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import * as plugins from '../../plugins.js';
|
|
2
|
+
import { DcRouterDb } from '../classes.dcrouter-db.js';
|
|
3
|
+
|
|
4
|
+
const DB_OPERATION_TIMEOUT_MS = 5_000;
|
|
5
|
+
|
|
6
|
+
const getDb = () => DcRouterDb.getInstance().getDb();
|
|
7
|
+
|
|
8
|
+
export interface IEmailTrafficBucketCounts {
|
|
9
|
+
sent: number;
|
|
10
|
+
received: number;
|
|
11
|
+
failed: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface IEmailTrafficBucketSnapshot extends IEmailTrafficBucketCounts {
|
|
15
|
+
bucketStart: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
@plugins.smartdata.Collection(() => getDb())
|
|
19
|
+
export class EmailTrafficBucketDoc extends plugins.smartdata.SmartDataDbDoc<
|
|
20
|
+
EmailTrafficBucketDoc,
|
|
21
|
+
EmailTrafficBucketDoc
|
|
22
|
+
> implements IEmailTrafficBucketSnapshot {
|
|
23
|
+
@plugins.smartdata.unI()
|
|
24
|
+
@plugins.smartdata.svDb()
|
|
25
|
+
public id!: string;
|
|
26
|
+
|
|
27
|
+
@plugins.smartdata.svDb()
|
|
28
|
+
public bucketStart!: number;
|
|
29
|
+
|
|
30
|
+
@plugins.smartdata.svDb()
|
|
31
|
+
public sent: number = 0;
|
|
32
|
+
|
|
33
|
+
@plugins.smartdata.svDb()
|
|
34
|
+
public received: number = 0;
|
|
35
|
+
|
|
36
|
+
@plugins.smartdata.svDb()
|
|
37
|
+
public failed: number = 0;
|
|
38
|
+
|
|
39
|
+
@plugins.smartdata.svDb()
|
|
40
|
+
public createdAt!: number;
|
|
41
|
+
|
|
42
|
+
@plugins.smartdata.svDb()
|
|
43
|
+
public updatedAt!: number;
|
|
44
|
+
|
|
45
|
+
constructor() {
|
|
46
|
+
super();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
private static async getNativeCollection() {
|
|
50
|
+
const smartdataCollection = (EmailTrafficBucketDoc as typeof EmailTrafficBucketDoc & {
|
|
51
|
+
collection: plugins.smartdata.SmartdataCollection<EmailTrafficBucketDoc>;
|
|
52
|
+
}).collection;
|
|
53
|
+
await smartdataCollection.init();
|
|
54
|
+
const probe = new EmailTrafficBucketDoc();
|
|
55
|
+
await smartdataCollection.markUniqueIndexes(probe.uniqueIndexes || []);
|
|
56
|
+
await smartdataCollection.createRegularIndexes(probe.regularIndexes || []);
|
|
57
|
+
return smartdataCollection.mongoDbCollection;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
private static validateSnapshot(snapshotArg: IEmailTrafficBucketSnapshot): void {
|
|
61
|
+
const counts = [snapshotArg.sent, snapshotArg.received, snapshotArg.failed];
|
|
62
|
+
if (
|
|
63
|
+
!Number.isSafeInteger(snapshotArg.bucketStart)
|
|
64
|
+
|| snapshotArg.bucketStart < 0
|
|
65
|
+
|| snapshotArg.bucketStart % 60_000 !== 0
|
|
66
|
+
|| counts.some((count) => !Number.isSafeInteger(count) || count < 0)
|
|
67
|
+
) {
|
|
68
|
+
throw new Error('Invalid email traffic bucket snapshot');
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Persist retry-safe absolute monotonic counters. */
|
|
73
|
+
public static async persistAbsolute(snapshotsArg: IEmailTrafficBucketSnapshot[]): Promise<void> {
|
|
74
|
+
if (snapshotsArg.length === 0) return;
|
|
75
|
+
if (snapshotsArg.length > 500) {
|
|
76
|
+
throw new Error('EmailTrafficBucketDoc.persistAbsolute accepts at most 500 buckets');
|
|
77
|
+
}
|
|
78
|
+
for (const snapshot of snapshotsArg) {
|
|
79
|
+
EmailTrafficBucketDoc.validateSnapshot(snapshot);
|
|
80
|
+
}
|
|
81
|
+
const collection = await EmailTrafficBucketDoc.getNativeCollection();
|
|
82
|
+
const now = Date.now();
|
|
83
|
+
const updatedAt = new Date(now).toISOString();
|
|
84
|
+
await collection.bulkWrite(
|
|
85
|
+
snapshotsArg.map((snapshot) => ({
|
|
86
|
+
updateOne: {
|
|
87
|
+
filter: { bucketStart: snapshot.bucketStart },
|
|
88
|
+
update: {
|
|
89
|
+
$max: {
|
|
90
|
+
sent: snapshot.sent,
|
|
91
|
+
received: snapshot.received,
|
|
92
|
+
failed: snapshot.failed,
|
|
93
|
+
},
|
|
94
|
+
$set: {
|
|
95
|
+
updatedAt: now,
|
|
96
|
+
_updatedAt: updatedAt,
|
|
97
|
+
},
|
|
98
|
+
$setOnInsert: {
|
|
99
|
+
id: `email-traffic-${snapshot.bucketStart}`,
|
|
100
|
+
bucketStart: snapshot.bucketStart,
|
|
101
|
+
createdAt: now,
|
|
102
|
+
_createdAt: updatedAt,
|
|
103
|
+
},
|
|
104
|
+
},
|
|
105
|
+
upsert: true,
|
|
106
|
+
},
|
|
107
|
+
})),
|
|
108
|
+
{ ordered: false, timeoutMS: DB_OPERATION_TIMEOUT_MS },
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
public static async loadSince(cutoffArg: number): Promise<IEmailTrafficBucketSnapshot[]> {
|
|
113
|
+
if (!Number.isSafeInteger(cutoffArg) || cutoffArg < 0) {
|
|
114
|
+
throw new Error('EmailTrafficBucketDoc.loadSince requires a valid cutoff');
|
|
115
|
+
}
|
|
116
|
+
const collection = await EmailTrafficBucketDoc.getNativeCollection();
|
|
117
|
+
const cursor = collection
|
|
118
|
+
.find(
|
|
119
|
+
{ bucketStart: { $gte: cutoffArg } },
|
|
120
|
+
{ timeoutMS: DB_OPERATION_TIMEOUT_MS },
|
|
121
|
+
)
|
|
122
|
+
.sort({ bucketStart: 1 });
|
|
123
|
+
try {
|
|
124
|
+
const rows = await cursor.toArray();
|
|
125
|
+
return rows.map((row) => ({
|
|
126
|
+
bucketStart: Number(row.bucketStart),
|
|
127
|
+
sent: Number(row.sent || 0),
|
|
128
|
+
received: Number(row.received || 0),
|
|
129
|
+
failed: Number(row.failed || 0),
|
|
130
|
+
}));
|
|
131
|
+
} finally {
|
|
132
|
+
await cursor.close({ timeoutMS: DB_OPERATION_TIMEOUT_MS });
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
public static async pruneBefore(cutoffArg: number): Promise<number> {
|
|
137
|
+
if (!Number.isSafeInteger(cutoffArg) || cutoffArg < 0) {
|
|
138
|
+
throw new Error('EmailTrafficBucketDoc.pruneBefore requires a valid cutoff');
|
|
139
|
+
}
|
|
140
|
+
const collection = await EmailTrafficBucketDoc.getNativeCollection();
|
|
141
|
+
let deletedCount = 0;
|
|
142
|
+
while (true) {
|
|
143
|
+
const rows = await collection
|
|
144
|
+
.find(
|
|
145
|
+
{ bucketStart: { $lt: cutoffArg } },
|
|
146
|
+
{ projection: { _id: 1 }, timeoutMS: DB_OPERATION_TIMEOUT_MS },
|
|
147
|
+
)
|
|
148
|
+
.sort({ bucketStart: 1, _id: 1 })
|
|
149
|
+
.limit(500)
|
|
150
|
+
.toArray();
|
|
151
|
+
if (rows.length === 0) break;
|
|
152
|
+
const result = await collection.deleteMany(
|
|
153
|
+
{ _id: { $in: rows.map((row) => row._id) } },
|
|
154
|
+
{ timeoutMS: DB_OPERATION_TIMEOUT_MS },
|
|
155
|
+
);
|
|
156
|
+
deletedCount += result.deletedCount;
|
|
157
|
+
if (rows.length < 500) break;
|
|
158
|
+
}
|
|
159
|
+
return deletedCount;
|
|
160
|
+
}
|
|
161
|
+
}
|
package/ts/db/documents/index.ts
CHANGED
|
@@ -4,6 +4,8 @@ export * from './classes.cached.ip.reputation.js';
|
|
|
4
4
|
export * from './classes.ip-intelligence.doc.js';
|
|
5
5
|
export * from './classes.security-block-rule.doc.js';
|
|
6
6
|
export * from './classes.security-policy-audit.doc.js';
|
|
7
|
+
export * from './classes.authentication-event.doc.js';
|
|
8
|
+
export * from './classes.email-traffic-bucket.doc.js';
|
|
7
9
|
|
|
8
10
|
// Config document classes
|
|
9
11
|
export * from './classes.route.doc.js';
|
|
@@ -8,10 +8,13 @@ import type {
|
|
|
8
8
|
import { AcceptEnvelopeRejectionError } from '@push.rocks/smartmta';
|
|
9
9
|
import type {
|
|
10
10
|
Email,
|
|
11
|
+
IAcceptedEnvelopeDispatchMetadata,
|
|
12
|
+
IAcceptedEnvelopeRecipientPlan,
|
|
11
13
|
IAcceptEnvelopeContext,
|
|
12
14
|
IExtendedSmtpSession,
|
|
13
15
|
IMessageAcceptanceContext,
|
|
14
16
|
IMessageAcceptanceDecision,
|
|
17
|
+
IResolvedRecipientRoute,
|
|
15
18
|
UnifiedEmailServer,
|
|
16
19
|
} from '@push.rocks/smartmta';
|
|
17
20
|
import type { DcRouter } from '../classes.dcrouter.js';
|
|
@@ -76,6 +79,9 @@ const ACCEPTED_EMAIL_SPOOL_BATCH_SIZE = 25;
|
|
|
76
79
|
const ACCEPTED_EMAIL_STOP_DRAIN_TIMEOUT_MS = 30_000;
|
|
77
80
|
/** Retention for catch-all stored inbound mail (30 days). */
|
|
78
81
|
const INBOUND_STORE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
|
|
82
|
+
/** Redispatch attempts for the non-store recipients of a durable envelope. */
|
|
83
|
+
const ENVELOPE_DISPATCH_MAX_ATTEMPTS = 10;
|
|
84
|
+
const ENVELOPE_DISPATCH_RETRY_DELAY_MS = 5 * 60_000;
|
|
79
85
|
|
|
80
86
|
/**
|
|
81
87
|
* Permanent per-email storage failure: the raw RFC822 payload of an accepted
|
|
@@ -136,8 +142,32 @@ type TStoredCachedEmailSession = {
|
|
|
136
142
|
};
|
|
137
143
|
};
|
|
138
144
|
|
|
145
|
+
/**
|
|
146
|
+
* Persisted per-recipient dispatch state for a durably accepted envelope.
|
|
147
|
+
* Holds everything needed to replay `dispatchAcceptedEnvelope` byte-identically:
|
|
148
|
+
* upstream fingerprints each recipient over the raw message, the metadata and
|
|
149
|
+
* the plan entry, so a replay reconstructed from anything else would be refused
|
|
150
|
+
* as an idempotency-key reuse instead of retrying.
|
|
151
|
+
*/
|
|
152
|
+
type TStoredEnvelopeDispatch = {
|
|
153
|
+
plan: IAcceptedEnvelopeRecipientPlan[];
|
|
154
|
+
metadata: IAcceptedEnvelopeDispatchMetadata;
|
|
155
|
+
attempts: number;
|
|
156
|
+
results?: Array<{
|
|
157
|
+
recipient: string;
|
|
158
|
+
status: string;
|
|
159
|
+
message?: string;
|
|
160
|
+
smtpCode?: number;
|
|
161
|
+
}>;
|
|
162
|
+
pendingRecipients?: string[];
|
|
163
|
+
/** Set once retries are exhausted; the row stops being redispatched. */
|
|
164
|
+
abandonedAt?: string;
|
|
165
|
+
};
|
|
166
|
+
|
|
139
167
|
type TStoredCachedEmailRouteData = {
|
|
168
|
+
acceptance?: string;
|
|
140
169
|
session?: TStoredCachedEmailSession;
|
|
170
|
+
envelopeDispatch?: TStoredEnvelopeDispatch;
|
|
141
171
|
smartMta?: {
|
|
142
172
|
status?: 'queued' | 'deferred' | 'delivered' | 'failed';
|
|
143
173
|
nextAttempt?: string;
|
|
@@ -185,6 +215,54 @@ export class AcceptedEmailSpool {
|
|
|
185
215
|
|
|
186
216
|
constructor(private dcRouterRef: DcRouter) {}
|
|
187
217
|
|
|
218
|
+
/**
|
|
219
|
+
* Direction is decided by the RECIPIENT, never by whether the session
|
|
220
|
+
* authenticated. Authentication grants permission to relay; it does not make
|
|
221
|
+
* a message addressed to a mailbox we host into outbound mail. A message with
|
|
222
|
+
* at least one locally hosted recipient is inbound — a local mailbox receives
|
|
223
|
+
* it — and only an envelope addressed exclusively to remote recipients is
|
|
224
|
+
* outbound.
|
|
225
|
+
*
|
|
226
|
+
* `IResolvedRecipientRoute.localDomain` is smartmta's own per-recipient
|
|
227
|
+
* verdict (non-null exactly when the domain registry hosts the domain), so
|
|
228
|
+
* the classification uses the same truth the routing decision used. When an
|
|
229
|
+
* acceptance context carries no resolution the envelope recipients are
|
|
230
|
+
* classified against the live registry instead — never against the session,
|
|
231
|
+
* which is the mistake being fixed.
|
|
232
|
+
*/
|
|
233
|
+
private deriveDirectionFromResolvedRoutes(
|
|
234
|
+
resolvedRecipientRoutes: readonly IResolvedRecipientRoute[] | undefined,
|
|
235
|
+
envelopeRecipientsArg: readonly string[],
|
|
236
|
+
): TCachedEmailDirection {
|
|
237
|
+
if (resolvedRecipientRoutes?.length) {
|
|
238
|
+
return resolvedRecipientRoutes.some((resolution) => !!resolution.localDomain)
|
|
239
|
+
? 'inbound'
|
|
240
|
+
: 'outbound';
|
|
241
|
+
}
|
|
242
|
+
return this.deriveDirectionFromRecipients(envelopeRecipientsArg);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Recipient-derived direction for programmatic submitters, which have no SMTP
|
|
247
|
+
* recipient resolution to consult. Falls back to the live domain registry.
|
|
248
|
+
*/
|
|
249
|
+
private deriveDirectionFromRecipients(recipientsArg: readonly string[]): TCachedEmailDirection {
|
|
250
|
+
const domainRegistry = this.dcRouterRef.emailServer?.domainRegistry;
|
|
251
|
+
if (!domainRegistry) {
|
|
252
|
+
// Without the registry there is no recipient truth to classify against.
|
|
253
|
+
// Programmatic submission is relay by construction, so outbound is the
|
|
254
|
+
// honest label rather than guessing from the session.
|
|
255
|
+
return 'outbound';
|
|
256
|
+
}
|
|
257
|
+
for (const recipient of recipientsArg) {
|
|
258
|
+
const domain = recipient.split('@')[1]?.trim().toLowerCase();
|
|
259
|
+
if (domain && domainRegistry.isDomainRegistered(domain)) {
|
|
260
|
+
return 'inbound';
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return 'outbound';
|
|
264
|
+
}
|
|
265
|
+
|
|
188
266
|
public async acceptMessage(
|
|
189
267
|
context: IMessageAcceptanceContext,
|
|
190
268
|
processAfterAccept = true,
|
|
@@ -218,7 +296,10 @@ export class AcceptedEmailSpool {
|
|
|
218
296
|
await this.persistRawMessage(cachedEmail, persistedRawMessage);
|
|
219
297
|
cachedEmail.status = 'pending';
|
|
220
298
|
cachedEmail.nextAttempt = new Date();
|
|
221
|
-
cachedEmail.direction =
|
|
299
|
+
cachedEmail.direction = this.deriveDirectionFromResolvedRoutes(
|
|
300
|
+
context.resolvedRecipientRoutes,
|
|
301
|
+
envelopeRecipients.length > 0 ? envelopeRecipients : cachedEmail.to,
|
|
302
|
+
);
|
|
222
303
|
cachedEmail.acceptedAt = Date.now();
|
|
223
304
|
cachedEmail.routeData = JSON.stringify({
|
|
224
305
|
acceptedAt: new Date().toISOString(),
|
|
@@ -256,6 +337,7 @@ export class AcceptedEmailSpool {
|
|
|
256
337
|
await this.notifyEmailQueuePersisted(cachedEmail, 'email-accepted-without-queue');
|
|
257
338
|
}
|
|
258
339
|
|
|
340
|
+
this.trackAcceptedInboundEmail(cachedEmail);
|
|
259
341
|
return {
|
|
260
342
|
accepted: true,
|
|
261
343
|
smtpCode: 250,
|
|
@@ -287,7 +369,7 @@ export class AcceptedEmailSpool {
|
|
|
287
369
|
cachedEmail.status = 'pending';
|
|
288
370
|
cachedEmail.nextAttempt = new Date();
|
|
289
371
|
cachedEmail.direction = optionsArg.direction
|
|
290
|
-
?? (optionsArg.
|
|
372
|
+
?? this.deriveDirectionFromRecipients(optionsArg.envelope.rcptTo);
|
|
291
373
|
cachedEmail.acceptedAt = Date.now();
|
|
292
374
|
cachedEmail.routeData = JSON.stringify({
|
|
293
375
|
acceptedAt: new Date().toISOString(),
|
|
@@ -319,6 +401,7 @@ export class AcceptedEmailSpool {
|
|
|
319
401
|
await this.notifyEmailQueuePersisted(cachedEmail, 'raw-email-accepted-without-queue');
|
|
320
402
|
}
|
|
321
403
|
|
|
404
|
+
this.trackAcceptedInboundEmail(cachedEmail);
|
|
322
405
|
return {
|
|
323
406
|
accepted: true,
|
|
324
407
|
spoolItemId: cachedEmail.id,
|
|
@@ -377,7 +460,10 @@ export class AcceptedEmailSpool {
|
|
|
377
460
|
cachedEmail.id,
|
|
378
461
|
);
|
|
379
462
|
await this.persistRawMessage(cachedEmail, persistedRawMessage);
|
|
380
|
-
cachedEmail.direction =
|
|
463
|
+
cachedEmail.direction = this.deriveDirectionFromResolvedRoutes(
|
|
464
|
+
context.resolvedRecipientRoutes,
|
|
465
|
+
context.envelope.rcptTo,
|
|
466
|
+
);
|
|
381
467
|
cachedEmail.acceptedAt = startedAtMs;
|
|
382
468
|
|
|
383
469
|
if (acceptance.dmarcReject) {
|
|
@@ -413,6 +499,15 @@ export class AcceptedEmailSpool {
|
|
|
413
499
|
} else {
|
|
414
500
|
cachedEmail.status = 'accepted';
|
|
415
501
|
}
|
|
502
|
+
if (nonStoreEntries.length > 0) {
|
|
503
|
+
// Relay recipients are dispatched after this row is committed. Park the
|
|
504
|
+
// row in a status the spool actually scans so a crash between the commit
|
|
505
|
+
// and the dispatch leaves those recipients recoverable instead of stranded
|
|
506
|
+
// in a terminal-looking 'accepted' row. The dispatch outcome settles it
|
|
507
|
+
// straight back to the acceptance status.
|
|
508
|
+
cachedEmail.status = 'deferred';
|
|
509
|
+
cachedEmail.nextAttempt = new Date(Date.now() + ENVELOPE_DISPATCH_RETRY_DELAY_MS);
|
|
510
|
+
}
|
|
416
511
|
cachedEmail.deliveredAt = new Date();
|
|
417
512
|
cachedEmail.setTTL(INBOUND_STORE_RETENTION_MS);
|
|
418
513
|
cachedEmail.appendSmtpTransaction(this.buildInboundTransaction(context, cachedEmail.id, {
|
|
@@ -422,6 +517,7 @@ export class AcceptedEmailSpool {
|
|
|
422
517
|
doubts: acceptance.doubts,
|
|
423
518
|
startedAtMs,
|
|
424
519
|
}));
|
|
520
|
+
const dispatchMetadata = this.buildEnvelopeDispatchMetadata(context);
|
|
425
521
|
cachedEmail.routeData = JSON.stringify({
|
|
426
522
|
acceptedAt: new Date(startedAtMs).toISOString(),
|
|
427
523
|
acceptance: 'durable-envelope',
|
|
@@ -432,6 +528,18 @@ export class AcceptedEmailSpool {
|
|
|
432
528
|
source: entry.source,
|
|
433
529
|
actionType: entry.action.type,
|
|
434
530
|
})),
|
|
531
|
+
// Persisted BEFORE the SMTP 250, so a dispatch failure for the non-store
|
|
532
|
+
// recipients is recoverable instead of being lost with the process.
|
|
533
|
+
...(nonStoreEntries.length > 0
|
|
534
|
+
? {
|
|
535
|
+
envelopeDispatch: {
|
|
536
|
+
plan: [...plan],
|
|
537
|
+
metadata: dispatchMetadata,
|
|
538
|
+
attempts: 0,
|
|
539
|
+
pendingRecipients: nonStoreEntries.map((entry) => entry.recipient),
|
|
540
|
+
} satisfies TStoredEnvelopeDispatch,
|
|
541
|
+
}
|
|
542
|
+
: {}),
|
|
435
543
|
session: {
|
|
436
544
|
id: session.id,
|
|
437
545
|
remoteAddress: session.remoteAddress,
|
|
@@ -459,27 +567,198 @@ export class AcceptedEmailSpool {
|
|
|
459
567
|
}
|
|
460
568
|
|
|
461
569
|
if (nonStoreEntries.length > 0) {
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
570
|
+
await this.dispatchEnvelopeRecipients(cachedEmail, persistedRawMessage, emailServer);
|
|
571
|
+
}
|
|
572
|
+
this.trackAcceptedInboundEmail(cachedEmail);
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
/**
|
|
576
|
+
* Session metadata for `dispatchAcceptedEnvelope`. It feeds the per-recipient
|
|
577
|
+
* idempotency fingerprint, so it must be JSON-round-trip stable: a replay
|
|
578
|
+
* reconstructed from persisted state has to produce byte-identical metadata
|
|
579
|
+
* or upstream refuses it as an idempotency-key reuse.
|
|
580
|
+
*/
|
|
581
|
+
private buildEnvelopeDispatchMetadata(
|
|
582
|
+
context: IAcceptEnvelopeContext,
|
|
583
|
+
): IAcceptedEnvelopeDispatchMetadata {
|
|
584
|
+
const session = context.session;
|
|
585
|
+
return {
|
|
586
|
+
mailFrom: context.envelope.mailFrom,
|
|
587
|
+
session: {
|
|
588
|
+
id: session.id || '',
|
|
589
|
+
remoteAddress: session.remoteAddress || '',
|
|
590
|
+
clientHostname: session.clientHostname || '',
|
|
591
|
+
secure: !!session.secure,
|
|
592
|
+
authenticated: !!session.authenticated,
|
|
593
|
+
},
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
/**
|
|
598
|
+
* Dispatch (or redispatch) the non-store recipients of a durably accepted
|
|
599
|
+
* envelope.
|
|
600
|
+
*
|
|
601
|
+
* The raw bytes handed to upstream are the exact bytes persisted for this row,
|
|
602
|
+
* because upstream fingerprints every recipient over the raw message: a later
|
|
603
|
+
* replay with different bytes would be rejected as an idempotency-key reuse
|
|
604
|
+
* rather than retried. Recipients that already succeeded are short-circuited
|
|
605
|
+
* by upstream's checkpoints, and `failed` results are deliberately not
|
|
606
|
+
* checkpointed upstream, so an identical replay retries exactly those.
|
|
607
|
+
*
|
|
608
|
+
* A `failed` recipient leaves the row non-terminal so the spool retries it;
|
|
609
|
+
* a `rejected` recipient is a permanent per-recipient refusal and is recorded
|
|
610
|
+
* durably instead of being retried. Either way the outcome is persisted — a
|
|
611
|
+
* relay recipient is never silently dropped after the SMTP 250.
|
|
612
|
+
*/
|
|
613
|
+
private async dispatchEnvelopeRecipients(
|
|
614
|
+
cachedEmailArg: CachedEmail,
|
|
615
|
+
rawMessageArg: string,
|
|
616
|
+
emailServerArg: UnifiedEmailServer,
|
|
617
|
+
): Promise<void> {
|
|
618
|
+
const routeData = this.parseCachedEmailRouteData(cachedEmailArg);
|
|
619
|
+
const envelopeDispatch = routeData.envelopeDispatch;
|
|
620
|
+
if (!envelopeDispatch || envelopeDispatch.abandonedAt) return;
|
|
621
|
+
|
|
622
|
+
const attempts = (envelopeDispatch.attempts || 0) + 1;
|
|
623
|
+
let dispatchResults: Array<{
|
|
624
|
+
recipient: string;
|
|
625
|
+
status: string;
|
|
626
|
+
message?: string;
|
|
627
|
+
smtpCode?: number;
|
|
628
|
+
}>;
|
|
629
|
+
try {
|
|
630
|
+
const dispatchResult = await emailServerArg.dispatchAcceptedEnvelope(
|
|
631
|
+
plugins.buffer.Buffer.from(rawMessageArg, 'utf8'),
|
|
632
|
+
envelopeDispatch.plan,
|
|
633
|
+
envelopeDispatch.metadata,
|
|
478
634
|
);
|
|
479
|
-
|
|
480
|
-
|
|
635
|
+
dispatchResults = dispatchResult.results.map((result) => ({
|
|
636
|
+
recipient: result.recipient,
|
|
637
|
+
status: result.status,
|
|
638
|
+
...(result.message ? { message: result.message } : {}),
|
|
639
|
+
...(result.smtpCode !== undefined ? { smtpCode: result.smtpCode } : {}),
|
|
640
|
+
}));
|
|
641
|
+
} catch (error: unknown) {
|
|
642
|
+
// A whole-call failure (missing managed queue storage, malformed
|
|
643
|
+
// checkpoint) is retried the same way a per-recipient failure is.
|
|
644
|
+
dispatchResults = envelopeDispatch.plan
|
|
645
|
+
.filter((entry) => entry.action.type !== 'store' && entry.action.type !== 'deliver')
|
|
646
|
+
.map((entry) => ({
|
|
647
|
+
recipient: entry.recipient,
|
|
648
|
+
status: 'failed',
|
|
649
|
+
message: (error as Error).message,
|
|
650
|
+
}));
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
const retryable = dispatchResults.filter((result) => result.status === 'failed');
|
|
654
|
+
const rejected = dispatchResults.filter((result) => result.status === 'rejected');
|
|
655
|
+
const exhausted = retryable.length > 0 && attempts >= ENVELOPE_DISPATCH_MAX_ATTEMPTS;
|
|
656
|
+
|
|
657
|
+
await this.persistEnvelopeDispatchOutcome(cachedEmailArg.id, {
|
|
658
|
+
attempts,
|
|
659
|
+
results: dispatchResults,
|
|
660
|
+
pendingRecipients: exhausted ? [] : retryable.map((result) => result.recipient),
|
|
661
|
+
abandoned: exhausted,
|
|
662
|
+
});
|
|
663
|
+
|
|
664
|
+
if (rejected.length > 0) {
|
|
665
|
+
logger.log('error', `Durable envelope ${cachedEmailArg.id}: ${rejected.length} relay recipient(s) permanently refused: ${rejected.map((result) => `${result.recipient}=${result.smtpCode || 550} ${result.message || 'rejected'}`).join('; ')}`);
|
|
666
|
+
}
|
|
667
|
+
if (exhausted) {
|
|
668
|
+
logger.log('error', `Durable envelope ${cachedEmailArg.id}: giving up on ${retryable.length} relay recipient(s) after ${attempts} dispatch attempts: ${retryable.map((result) => `${result.recipient}=${result.message || 'failed'}`).join('; ')}`);
|
|
669
|
+
} else if (retryable.length > 0) {
|
|
670
|
+
logger.log('warn', `Durable envelope ${cachedEmailArg.id}: ${retryable.length} relay recipient(s) failed dispatch (attempt ${attempts}/${ENVELOPE_DISPATCH_MAX_ATTEMPTS}), scheduled for redispatch: ${retryable.map((result) => `${result.recipient}=${result.message || 'failed'}`).join('; ')}`);
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* Persist a dispatch attempt's outcome on the durable-envelope row.
|
|
676
|
+
*
|
|
677
|
+
* The row's status tracks the stored envelope, not the relay: it goes
|
|
678
|
+
* `deferred` only while relay recipients still need a redispatch, and returns
|
|
679
|
+
* to its acceptance status once none do. Relay progress itself lives in
|
|
680
|
+
* `routeData.envelopeDispatch`, so a relay failure can never mark a row whose
|
|
681
|
+
* local copy stored successfully as failed.
|
|
682
|
+
*/
|
|
683
|
+
private async persistEnvelopeDispatchOutcome(
|
|
684
|
+
cachedEmailIdArg: string,
|
|
685
|
+
outcomeArg: {
|
|
686
|
+
attempts: number;
|
|
687
|
+
results: Array<{ recipient: string; status: string; message?: string; smtpCode?: number }>;
|
|
688
|
+
pendingRecipients: string[];
|
|
689
|
+
abandoned: boolean;
|
|
690
|
+
},
|
|
691
|
+
): Promise<void> {
|
|
692
|
+
await this.runCachedEmailUpdate(cachedEmailIdArg, async () => {
|
|
693
|
+
const cachedEmail = await CachedEmail.findById(cachedEmailIdArg);
|
|
694
|
+
if (!cachedEmail) return;
|
|
695
|
+
const routeData = this.parseCachedEmailRouteData(cachedEmail);
|
|
696
|
+
if (!routeData.envelopeDispatch) return;
|
|
697
|
+
routeData.envelopeDispatch = {
|
|
698
|
+
...routeData.envelopeDispatch,
|
|
699
|
+
attempts: outcomeArg.attempts,
|
|
700
|
+
results: outcomeArg.results,
|
|
701
|
+
pendingRecipients: outcomeArg.pendingRecipients,
|
|
702
|
+
...(outcomeArg.abandoned ? { abandonedAt: new Date().toISOString() } : {}),
|
|
703
|
+
};
|
|
704
|
+
cachedEmail.routeData = JSON.stringify(routeData);
|
|
705
|
+
|
|
706
|
+
const stillPending = outcomeArg.pendingRecipients.length > 0;
|
|
707
|
+
const acceptanceStatus = cachedEmail.lastError && !stillPending ? 'flagged' : 'accepted';
|
|
708
|
+
if (stillPending) {
|
|
709
|
+
cachedEmail.status = 'deferred';
|
|
710
|
+
cachedEmail.nextAttempt = new Date(Date.now() + ENVELOPE_DISPATCH_RETRY_DELAY_MS);
|
|
711
|
+
} else if (cachedEmail.status === 'deferred') {
|
|
712
|
+
cachedEmail.status = acceptanceStatus;
|
|
713
|
+
cachedEmail.nextAttempt = new Date();
|
|
481
714
|
}
|
|
715
|
+
await cachedEmail.save();
|
|
716
|
+
await this.notifyEmailQueuePersisted(
|
|
717
|
+
cachedEmail,
|
|
718
|
+
stillPending ? 'envelope-dispatch-deferred' : 'envelope-dispatch-settled',
|
|
719
|
+
);
|
|
720
|
+
});
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
/**
|
|
724
|
+
* Take exclusive ownership of a durably accepted envelope row in the spool.
|
|
725
|
+
*
|
|
726
|
+
* Runs before live-queue postponement and before the normal spool handoff: a
|
|
727
|
+
* relay sibling that did enqueue would otherwise postpone this row forever,
|
|
728
|
+
* and the normal handoff would re-run route evaluation and duplicate a
|
|
729
|
+
* delivery the stored copy already fulfilled.
|
|
730
|
+
*/
|
|
731
|
+
private async handleDurableEnvelopeRow(
|
|
732
|
+
cachedEmailArg: CachedEmail,
|
|
733
|
+
emailServerArg: UnifiedEmailServer,
|
|
734
|
+
): Promise<boolean> {
|
|
735
|
+
if (!this.isDurableEnvelopeRow(cachedEmailArg)) return false;
|
|
736
|
+
const envelopeDispatch = this.parseCachedEmailRouteData(cachedEmailArg).envelopeDispatch;
|
|
737
|
+
|
|
738
|
+
if (envelopeDispatch?.pendingRecipients?.length && !envelopeDispatch.abandonedAt) {
|
|
739
|
+
const rawMessage = await this.readRawMessage(cachedEmailArg);
|
|
740
|
+
await this.dispatchEnvelopeRecipients(
|
|
741
|
+
cachedEmailArg,
|
|
742
|
+
rawMessage.toString('utf8'),
|
|
743
|
+
emailServerArg,
|
|
744
|
+
);
|
|
745
|
+
return true;
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
// Nothing left to dispatch: settle the row back onto its acceptance status
|
|
749
|
+
// so a fully dispatched envelope does not linger as deferred.
|
|
750
|
+
if (cachedEmailArg.status === 'deferred') {
|
|
751
|
+
cachedEmailArg.status = cachedEmailArg.lastError ? 'flagged' : 'accepted';
|
|
752
|
+
cachedEmailArg.nextAttempt = new Date();
|
|
753
|
+
await cachedEmailArg.save();
|
|
754
|
+
await this.notifyEmailQueuePersisted(cachedEmailArg, 'envelope-dispatch-settled');
|
|
482
755
|
}
|
|
756
|
+
return true;
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
private trackAcceptedInboundEmail(cachedEmailArg: CachedEmail): void {
|
|
760
|
+
if (cachedEmailArg.direction !== 'inbound') return;
|
|
761
|
+
this.dcRouterRef.metricsManager?.trackEmailReceived(cachedEmailArg.from);
|
|
483
762
|
}
|
|
484
763
|
|
|
485
764
|
/**
|
|
@@ -772,6 +1051,11 @@ export class AcceptedEmailSpool {
|
|
|
772
1051
|
break;
|
|
773
1052
|
}
|
|
774
1053
|
try {
|
|
1054
|
+
// Durable envelopes are owned by the accepted-envelope dispatch path,
|
|
1055
|
+
// never by the route-evaluating handoff below.
|
|
1056
|
+
if (await this.handleDurableEnvelopeRow(cachedEmail, emailServer)) {
|
|
1057
|
+
continue;
|
|
1058
|
+
}
|
|
775
1059
|
if (await this.postponeLiveSmartMtaOwnedEmail(cachedEmail, emailServer)) {
|
|
776
1060
|
continue;
|
|
777
1061
|
}
|
|
@@ -946,6 +1230,15 @@ export class AcceptedEmailSpool {
|
|
|
946
1230
|
this.appendSmtpTransaction(cachedEmail, transaction);
|
|
947
1231
|
}
|
|
948
1232
|
this.updateSmartMtaRouteData(cachedEmail, item, status);
|
|
1233
|
+
if (this.isDurableEnvelopeRow(cachedEmail)) {
|
|
1234
|
+
// The row represents the durably accepted and stored envelope; the queue
|
|
1235
|
+
// item only covers its relay recipients. Record the relay telemetry but
|
|
1236
|
+
// never let a relay outcome overwrite the stored copy's status — a failed
|
|
1237
|
+
// relay must not mark a successfully stored envelope as failed.
|
|
1238
|
+
await cachedEmail.save();
|
|
1239
|
+
await this.notifyEmailQueuePersisted(cachedEmail, `envelope-relay-${status}`);
|
|
1240
|
+
return;
|
|
1241
|
+
}
|
|
949
1242
|
if (status === 'delivered') {
|
|
950
1243
|
cachedEmail.markDelivered();
|
|
951
1244
|
} else if (status === 'failed') {
|
|
@@ -1008,6 +1301,11 @@ export class AcceptedEmailSpool {
|
|
|
1008
1301
|
return cachedEmail.status === 'delivered' || cachedEmail.status === 'failed';
|
|
1009
1302
|
}
|
|
1010
1303
|
|
|
1304
|
+
/** Whether this row was durably accepted through the envelope acceptance path. */
|
|
1305
|
+
private isDurableEnvelopeRow(cachedEmail: CachedEmail): boolean {
|
|
1306
|
+
return this.parseCachedEmailRouteData(cachedEmail).acceptance === 'durable-envelope';
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1011
1309
|
/** The CachedEmail row a live queue item was spooled from (via the cache-id header). */
|
|
1012
1310
|
public getCachedEmailIdFromQueueItem(item: TSmartMtaQueueItemLike): string | undefined {
|
|
1013
1311
|
return this.getHeaderValue(item.processingResult?.headers, DCROUTER_CACHE_ID_HEADER)
|
|
@@ -1128,7 +1128,10 @@ export class WorkAppMailManager {
|
|
|
1128
1128
|
spoolItem: {
|
|
1129
1129
|
id: cachedEmailArg.id,
|
|
1130
1130
|
owner,
|
|
1131
|
-
|
|
1131
|
+
// The persisted field is the single source of truth: it is derived from
|
|
1132
|
+
// the recipients at acceptance, whereas re-deriving from the session
|
|
1133
|
+
// username here reproduced the "authenticated means outbound" mistake.
|
|
1134
|
+
direction: cachedEmailArg.direction,
|
|
1132
1135
|
status,
|
|
1133
1136
|
envelope: {
|
|
1134
1137
|
mailFrom: cachedEmailArg.from,
|