@serve.zone/dcrouter 18.2.1 → 18.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deno.json +1 -1
- package/dist_serve/bundle.js +2986 -2954
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.dcrouter.js +4 -1
- package/dist_ts/db/documents/classes.cached.email.d.ts +61 -4
- package/dist_ts/db/documents/classes.cached.email.js +126 -9
- package/dist_ts/email/classes.accepted-email-spool.d.ts +1 -8
- package/dist_ts/email/classes.accepted-email-spool.js +18 -52
- package/dist_ts/email/classes.mail-egress-coordinator.js +10 -1
- package/dist_ts/email/helpers.email-log-traffic.d.ts +19 -0
- package/dist_ts/email/helpers.email-log-traffic.js +79 -0
- package/dist_ts/email/helpers.inbound-security.d.ts +8 -0
- package/dist_ts/email/helpers.inbound-security.js +35 -0
- package/dist_ts/email/index.d.ts +2 -0
- package/dist_ts/email/index.js +3 -1
- package/dist_ts/opsserver/handlers/email-ops.handler.d.ts +6 -0
- package/dist_ts/opsserver/handlers/email-ops.handler.js +162 -17
- package/dist_ts/remoteingress/classes.tunnel-manager.d.ts +4 -0
- package/dist_ts/remoteingress/classes.tunnel-manager.js +10 -1
- package/dist_ts_interfaces/requests/email-ops.d.ts +85 -0
- package/dist_ts_migrations/index.js +147 -14
- package/dist_ts_web/00_commitinfo_data.js +1 -1
- package/dist_ts_web/appstate/email-ops.d.ts +27 -1
- package/dist_ts_web/appstate/email-ops.js +100 -4
- package/dist_ts_web/appstate/runtime.js +11 -3
- package/dist_ts_web/elements/email/ops-view-email-security.d.ts +2 -0
- package/dist_ts_web/elements/email/ops-view-email-security.js +58 -16
- package/dist_ts_web/elements/email/ops-view-emails.d.ts +14 -2
- package/dist_ts_web/elements/email/ops-view-emails.js +128 -4
- package/package.json +6 -6
- package/readme.md +19 -0
- package/readme.plan.md +22 -0
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.dcrouter.ts +7 -0
- package/ts/db/documents/classes.cached.email.ts +199 -7
- package/ts/email/classes.accepted-email-spool.ts +25 -66
- package/ts/email/classes.mail-egress-coordinator.ts +8 -0
- package/ts/email/helpers.email-log-traffic.ts +117 -0
- package/ts/email/helpers.inbound-security.ts +45 -0
- package/ts/email/index.ts +2 -0
- package/ts/opsserver/handlers/email-ops.handler.ts +214 -15
- package/ts/remoteingress/classes.tunnel-manager.ts +14 -0
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/appstate/email-ops.ts +129 -3
- package/ts_web/appstate/runtime.ts +13 -2
- package/ts_web/elements/email/ops-view-email-security.ts +56 -16
- package/ts_web/elements/email/ops-view-emails.ts +130 -2
- package/ts_web/readme.md +7 -0
|
@@ -18,58 +18,11 @@ import type {
|
|
|
18
18
|
UnifiedEmailServer,
|
|
19
19
|
} from '@push.rocks/smartmta';
|
|
20
20
|
import type { DcRouter } from '../classes.dcrouter.js';
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
doubts: string[];
|
|
27
|
-
dmarcReject: boolean;
|
|
28
|
-
dmarcDomain?: string;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
export function evaluateInboundAcceptance(securityResults: any): IInboundAcceptanceEvaluation {
|
|
32
|
-
const doubts: string[] = [];
|
|
33
|
-
let dmarcReject = false;
|
|
34
|
-
let dmarcDomain: string | undefined;
|
|
35
|
-
|
|
36
|
-
// Infrastructure failures (temperror, resolver problems) are OUR problem,
|
|
37
|
-
// not evidence against the sender — they never produce doubts.
|
|
38
|
-
const spfResult = String(securityResults?.spf?.result || '').toLowerCase();
|
|
39
|
-
if (spfResult === 'fail' || spfResult === 'softfail') {
|
|
40
|
-
doubts.push(`SPF ${spfResult} for ${securityResults?.spf?.domain || 'sender domain'}`);
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
// The verdict bridge emits a placeholder entry (status 'none') when a
|
|
44
|
-
// message carries no DKIM signature at all; only real failed signatures
|
|
45
|
-
// count as doubts.
|
|
46
|
-
const dkimSignatures = (Array.isArray(securityResults?.dkim) ? securityResults.dkim : [])
|
|
47
|
-
.filter((signature: any) => String(signature?.status || '').toLowerCase() !== 'none');
|
|
48
|
-
if (dkimSignatures.length > 0 && !dkimSignatures.some((signature: any) => signature?.is_valid)) {
|
|
49
|
-
doubts.push('DKIM signature(s) present but none verified');
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
// DMARC only matters when the sender domain publishes an enforcing policy;
|
|
53
|
-
// absent records surface as policy 'none' and are not doubts.
|
|
54
|
-
const dmarc = securityResults?.dmarc;
|
|
55
|
-
if (dmarc && dmarc.passed === false) {
|
|
56
|
-
const policy = String(dmarc.policy || '').toLowerCase();
|
|
57
|
-
const action = String(dmarc.action || '').toLowerCase();
|
|
58
|
-
if (policy === 'reject' || policy === 'quarantine' || action === 'reject' || action === 'quarantine') {
|
|
59
|
-
dmarcDomain = dmarc.domain || undefined;
|
|
60
|
-
if (action === 'reject' || policy === 'reject') {
|
|
61
|
-
dmarcReject = true;
|
|
62
|
-
}
|
|
63
|
-
doubts.push(`DMARC failed for ${dmarc.domain || 'sender domain'} (policy ${dmarc.policy || 'unknown'})`);
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
return {
|
|
67
|
-
status: doubts.length > 0 ? 'flagged' : 'accepted',
|
|
68
|
-
doubts,
|
|
69
|
-
dmarcReject,
|
|
70
|
-
dmarcDomain,
|
|
71
|
-
};
|
|
72
|
-
}
|
|
21
|
+
import { evaluateInboundAcceptance } from './helpers.inbound-security.js';
|
|
22
|
+
export {
|
|
23
|
+
evaluateInboundAcceptance,
|
|
24
|
+
type IInboundAcceptanceEvaluation,
|
|
25
|
+
} from './helpers.inbound-security.js';
|
|
73
26
|
|
|
74
27
|
export const DCROUTER_CACHE_ID_HEADER = 'X-Dcrouter-Cached-Email-Id';
|
|
75
28
|
const ACCEPTED_EMAIL_SPOOL_INTERVAL_MS = 60_000;
|
|
@@ -452,7 +405,21 @@ export class AcceptedEmailSpool {
|
|
|
452
405
|
const plan = emailServer.createAcceptedEnvelopeDispatchPlan(context, idempotencyKeys);
|
|
453
406
|
const nonStoreEntries = plan.filter((entry) => entry.action.type !== 'store');
|
|
454
407
|
|
|
455
|
-
|
|
408
|
+
cachedEmail.direction = this.deriveDirectionFromResolvedRoutes(
|
|
409
|
+
context.resolvedRecipientRoutes,
|
|
410
|
+
context.envelope.rcptTo,
|
|
411
|
+
);
|
|
412
|
+
// Submission/relay authentication is not a receiver verdict. Only locally
|
|
413
|
+
// received mail is classified or enforced here; outbound mail gets its
|
|
414
|
+
// post-signing, actual-egress self-check from the delivery transaction.
|
|
415
|
+
const acceptance = evaluateInboundAcceptance(
|
|
416
|
+
cachedEmail.direction === 'inbound' ? context.securityResults : null,
|
|
417
|
+
);
|
|
418
|
+
if (cachedEmail.direction === 'inbound' && context.securityResults) {
|
|
419
|
+
cachedEmail.inboundSecurityDisposition = acceptance.dmarcReject
|
|
420
|
+
? 'rejected'
|
|
421
|
+
: acceptance.status;
|
|
422
|
+
}
|
|
456
423
|
const startedAtMs = Date.now();
|
|
457
424
|
|
|
458
425
|
const persistedRawMessage = this.setDcRouterCacheIdHeader(
|
|
@@ -460,10 +427,6 @@ export class AcceptedEmailSpool {
|
|
|
460
427
|
cachedEmail.id,
|
|
461
428
|
);
|
|
462
429
|
await this.persistRawMessage(cachedEmail, persistedRawMessage);
|
|
463
|
-
cachedEmail.direction = this.deriveDirectionFromResolvedRoutes(
|
|
464
|
-
context.resolvedRecipientRoutes,
|
|
465
|
-
context.envelope.rcptTo,
|
|
466
|
-
);
|
|
467
430
|
cachedEmail.acceptedAt = startedAtMs;
|
|
468
431
|
|
|
469
432
|
if (acceptance.dmarcReject) {
|
|
@@ -493,12 +456,9 @@ export class AcceptedEmailSpool {
|
|
|
493
456
|
throw new AcceptEnvelopeRejectionError(550, rejectionMessage);
|
|
494
457
|
}
|
|
495
458
|
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
} else {
|
|
500
|
-
cachedEmail.status = 'accepted';
|
|
501
|
-
}
|
|
459
|
+
// Security disposition is orthogonal to the delivery lifecycle. The Ops
|
|
460
|
+
// API synthesizes the visible flagged badge from inboundSecurityDisposition.
|
|
461
|
+
cachedEmail.status = 'accepted';
|
|
502
462
|
if (nonStoreEntries.length > 0) {
|
|
503
463
|
// Relay recipients are dispatched after this row is committed. Park the
|
|
504
464
|
// row in a status the spool actually scans so a crash between the commit
|
|
@@ -704,12 +664,11 @@ export class AcceptedEmailSpool {
|
|
|
704
664
|
cachedEmail.routeData = JSON.stringify(routeData);
|
|
705
665
|
|
|
706
666
|
const stillPending = outcomeArg.pendingRecipients.length > 0;
|
|
707
|
-
const acceptanceStatus = cachedEmail.lastError && !stillPending ? 'flagged' : 'accepted';
|
|
708
667
|
if (stillPending) {
|
|
709
668
|
cachedEmail.status = 'deferred';
|
|
710
669
|
cachedEmail.nextAttempt = new Date(Date.now() + ENVELOPE_DISPATCH_RETRY_DELAY_MS);
|
|
711
670
|
} else if (cachedEmail.status === 'deferred') {
|
|
712
|
-
cachedEmail.status =
|
|
671
|
+
cachedEmail.status = 'accepted';
|
|
713
672
|
cachedEmail.nextAttempt = new Date();
|
|
714
673
|
}
|
|
715
674
|
await cachedEmail.save();
|
|
@@ -748,7 +707,7 @@ export class AcceptedEmailSpool {
|
|
|
748
707
|
// Nothing left to dispatch: settle the row back onto its acceptance status
|
|
749
708
|
// so a fully dispatched envelope does not linger as deferred.
|
|
750
709
|
if (cachedEmailArg.status === 'deferred') {
|
|
751
|
-
cachedEmailArg.status =
|
|
710
|
+
cachedEmailArg.status = 'accepted';
|
|
752
711
|
cachedEmailArg.nextAttempt = new Date();
|
|
753
712
|
await cachedEmailArg.save();
|
|
754
713
|
await this.notifyEmailQueuePersisted(cachedEmailArg, 'envelope-dispatch-settled');
|
|
@@ -134,6 +134,14 @@ export class MailEgressCoordinator {
|
|
|
134
134
|
connectPort: proxy.listenPort,
|
|
135
135
|
poolKey: `remoteIngress:${proxy.edgeId}:${context.host}:${context.port}`,
|
|
136
136
|
ehloHostname: selected.hostname,
|
|
137
|
+
sourceEvidence: tunnelManager.waitForEgressConnectionEvidence(proxy.proxyId, proxy.edgeId)
|
|
138
|
+
.then((eventArg) => {
|
|
139
|
+
if (!eventArg?.sourceIp || !eventArg.addressFamily) return null;
|
|
140
|
+
return {
|
|
141
|
+
sourceIp: eventArg.sourceIp,
|
|
142
|
+
addressFamily: eventArg.addressFamily,
|
|
143
|
+
};
|
|
144
|
+
}),
|
|
137
145
|
close: async () => {
|
|
138
146
|
if (closed) {
|
|
139
147
|
return;
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
IEmail,
|
|
3
|
+
IEmailLogTraffic,
|
|
4
|
+
TEmailLogTrafficWindow,
|
|
5
|
+
} from '../../ts_interfaces/requests/email-ops.js';
|
|
6
|
+
|
|
7
|
+
const MINUTE_MS = 60_000;
|
|
8
|
+
const HOUR_MS = 60 * MINUTE_MS;
|
|
9
|
+
const DAY_MS = 24 * HOUR_MS;
|
|
10
|
+
export const EMAIL_LOG_RETENTION_MS = 30 * DAY_MS;
|
|
11
|
+
export const EMAIL_LOG_SEARCH_MAX_LENGTH = 200;
|
|
12
|
+
|
|
13
|
+
export interface IEmailLogTrafficWindowDescriptor {
|
|
14
|
+
window: TEmailLogTrafficWindow;
|
|
15
|
+
label: string;
|
|
16
|
+
durationMs: number;
|
|
17
|
+
bucketSizeMs: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface IEmailLogSparseTrafficBucket {
|
|
21
|
+
bucketStart: number;
|
|
22
|
+
sent: number;
|
|
23
|
+
received: number;
|
|
24
|
+
failed: number;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const trafficWindows: IEmailLogTrafficWindowDescriptor[] = [
|
|
28
|
+
{ window: '24h', label: 'Email Traffic — Last 24 Hours', durationMs: DAY_MS, bucketSizeMs: MINUTE_MS },
|
|
29
|
+
{ window: '7d', label: 'Email Traffic — Last 7 Days', durationMs: 7 * DAY_MS, bucketSizeMs: 15 * MINUTE_MS },
|
|
30
|
+
{ window: '14d', label: 'Email Traffic — Last 14 Days', durationMs: 14 * DAY_MS, bucketSizeMs: 30 * MINUTE_MS },
|
|
31
|
+
{ window: '30d', label: 'Email Traffic — Last 30 Days', durationMs: 30 * DAY_MS, bucketSizeMs: HOUR_MS },
|
|
32
|
+
];
|
|
33
|
+
|
|
34
|
+
export function normalizeEmailLogSearch(searchArg: unknown): string {
|
|
35
|
+
if (searchArg === undefined || searchArg === null) return '';
|
|
36
|
+
if (typeof searchArg !== 'string') {
|
|
37
|
+
throw new Error('Email log search must be a string');
|
|
38
|
+
}
|
|
39
|
+
const search = searchArg.trim();
|
|
40
|
+
if (search.length > EMAIL_LOG_SEARCH_MAX_LENGTH) {
|
|
41
|
+
throw new Error(`Email log search accepts at most ${EMAIL_LOG_SEARCH_MAX_LENGTH} characters`);
|
|
42
|
+
}
|
|
43
|
+
return search;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function selectEmailLogTrafficWindow(
|
|
47
|
+
searchArg: string,
|
|
48
|
+
oldestMatchAtArg: number | null,
|
|
49
|
+
nowArg = Date.now(),
|
|
50
|
+
): IEmailLogTrafficWindowDescriptor {
|
|
51
|
+
if (!searchArg || oldestMatchAtArg === null || !Number.isFinite(oldestMatchAtArg)) {
|
|
52
|
+
return trafficWindows[0];
|
|
53
|
+
}
|
|
54
|
+
return trafficWindows.find((window) => {
|
|
55
|
+
const lastBucketStart = Math.floor(nowArg / window.bucketSizeMs) * window.bucketSizeMs;
|
|
56
|
+
const windowStart = lastBucketStart - window.durationMs + window.bucketSizeMs;
|
|
57
|
+
return oldestMatchAtArg >= windowStart;
|
|
58
|
+
}) || trafficWindows.at(-1)!;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function buildEmailLogTraffic(
|
|
62
|
+
descriptorArg: IEmailLogTrafficWindowDescriptor,
|
|
63
|
+
sparseBucketsArg: IEmailLogSparseTrafficBucket[],
|
|
64
|
+
nowArg = Date.now(),
|
|
65
|
+
): IEmailLogTraffic {
|
|
66
|
+
const lastBucketStart = Math.floor(nowArg / descriptorArg.bucketSizeMs) * descriptorArg.bucketSizeMs;
|
|
67
|
+
const windowStart = lastBucketStart - descriptorArg.durationMs + descriptorArg.bucketSizeMs;
|
|
68
|
+
const bucketsByStart = new Map(
|
|
69
|
+
sparseBucketsArg.map((bucket) => [bucket.bucketStart, bucket]),
|
|
70
|
+
);
|
|
71
|
+
const sent: IEmailLogTraffic['sent'] = [];
|
|
72
|
+
const received: IEmailLogTraffic['received'] = [];
|
|
73
|
+
const failed: IEmailLogTraffic['failed'] = [];
|
|
74
|
+
for (
|
|
75
|
+
let timestamp = windowStart;
|
|
76
|
+
timestamp <= lastBucketStart;
|
|
77
|
+
timestamp += descriptorArg.bucketSizeMs
|
|
78
|
+
) {
|
|
79
|
+
const bucket = bucketsByStart.get(timestamp);
|
|
80
|
+
sent.push({ timestamp, value: Math.max(0, Number(bucket?.sent || 0)) });
|
|
81
|
+
received.push({ timestamp, value: Math.max(0, Number(bucket?.received || 0)) });
|
|
82
|
+
failed.push({ timestamp, value: Math.max(0, Number(bucket?.failed || 0)) });
|
|
83
|
+
}
|
|
84
|
+
return {
|
|
85
|
+
...descriptorArg,
|
|
86
|
+
windowStart,
|
|
87
|
+
windowEnd: nowArg,
|
|
88
|
+
sent,
|
|
89
|
+
received,
|
|
90
|
+
failed,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function aggregateEmailLogTrafficInMemory(
|
|
95
|
+
emailsArg: IEmail[],
|
|
96
|
+
descriptorArg: IEmailLogTrafficWindowDescriptor,
|
|
97
|
+
nowArg = Date.now(),
|
|
98
|
+
): IEmailLogSparseTrafficBucket[] {
|
|
99
|
+
const lastBucketStart = Math.floor(nowArg / descriptorArg.bucketSizeMs) * descriptorArg.bucketSizeMs;
|
|
100
|
+
const windowStart = lastBucketStart - descriptorArg.durationMs + descriptorArg.bucketSizeMs;
|
|
101
|
+
const buckets = new Map<number, IEmailLogSparseTrafficBucket>();
|
|
102
|
+
for (const email of emailsArg) {
|
|
103
|
+
const timestamp = new Date(email.timestamp).getTime();
|
|
104
|
+
if (!Number.isFinite(timestamp) || timestamp < windowStart || timestamp > nowArg) continue;
|
|
105
|
+
const bucketStart = Math.floor(timestamp / descriptorArg.bucketSizeMs) * descriptorArg.bucketSizeMs;
|
|
106
|
+
const bucket = buckets.get(bucketStart) || { bucketStart, sent: 0, received: 0, failed: 0 };
|
|
107
|
+
if (email.status === 'bounced' || email.status === 'rejected') {
|
|
108
|
+
bucket.failed++;
|
|
109
|
+
} else if (email.direction === 'outbound') {
|
|
110
|
+
bucket.sent++;
|
|
111
|
+
} else {
|
|
112
|
+
bucket.received++;
|
|
113
|
+
}
|
|
114
|
+
buckets.set(bucketStart, bucket);
|
|
115
|
+
}
|
|
116
|
+
return [...buckets.values()].sort((a, b) => a.bucketStart - b.bucketStart);
|
|
117
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/** Inbound authentication verdict evaluation shared by SMTP acceptance and ops reporting. */
|
|
2
|
+
export interface IInboundAcceptanceEvaluation {
|
|
3
|
+
status: 'accepted' | 'flagged';
|
|
4
|
+
doubts: string[];
|
|
5
|
+
dmarcReject: boolean;
|
|
6
|
+
dmarcDomain?: string;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export function evaluateInboundAcceptance(securityResults: any): IInboundAcceptanceEvaluation {
|
|
10
|
+
const doubts: string[] = [];
|
|
11
|
+
let dmarcReject = false;
|
|
12
|
+
let dmarcDomain: string | undefined;
|
|
13
|
+
|
|
14
|
+
// Infrastructure failures are our problem, not evidence against the sender.
|
|
15
|
+
const spfResult = String(securityResults?.spf?.result || '').toLowerCase();
|
|
16
|
+
if (spfResult === 'fail' || spfResult === 'softfail') {
|
|
17
|
+
doubts.push(`SPF ${spfResult} for ${securityResults?.spf?.domain || 'sender domain'}`);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// Ignore the placeholder emitted when no DKIM signature exists.
|
|
21
|
+
const dkimSignatures = (Array.isArray(securityResults?.dkim) ? securityResults.dkim : [])
|
|
22
|
+
.filter((signature: any) => String(signature?.status || '').toLowerCase() !== 'none');
|
|
23
|
+
if (dkimSignatures.length > 0 && !dkimSignatures.some((signature: any) => signature?.is_valid)) {
|
|
24
|
+
doubts.push('DKIM signature(s) present but none verified');
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// A failed DMARC alignment is a security finding even when the sender
|
|
28
|
+
// publishes p=none. Policy controls enforcement, not the authentication result.
|
|
29
|
+
const dmarc = securityResults?.dmarc;
|
|
30
|
+
if (dmarc && dmarc.passed === false) {
|
|
31
|
+
const policy = String(dmarc.policy || '').toLowerCase();
|
|
32
|
+
const action = String(dmarc.action || '').toLowerCase();
|
|
33
|
+
dmarcDomain = dmarc.domain || undefined;
|
|
34
|
+
if (action === 'reject' || policy === 'reject') {
|
|
35
|
+
dmarcReject = true;
|
|
36
|
+
}
|
|
37
|
+
doubts.push(`DMARC failed for ${dmarc.domain || 'sender domain'} (policy ${dmarc.policy || 'unknown'})`);
|
|
38
|
+
}
|
|
39
|
+
return {
|
|
40
|
+
status: doubts.length > 0 ? 'flagged' : 'accepted',
|
|
41
|
+
doubts,
|
|
42
|
+
dmarcReject,
|
|
43
|
+
dmarcDomain,
|
|
44
|
+
};
|
|
45
|
+
}
|
package/ts/email/index.ts
CHANGED
|
@@ -15,3 +15,5 @@ export * from './classes.smtp-account.manager.js';
|
|
|
15
15
|
export * from './smtp-scram.js';
|
|
16
16
|
export * from './classes.workapp-mail-manager.js';
|
|
17
17
|
export * from './email-dns-records.js';
|
|
18
|
+
export * from './helpers.email-log-traffic.js';
|
|
19
|
+
export * from './helpers.inbound-security.js';
|
|
@@ -3,6 +3,14 @@ import type { OpsServer } from '../classes.opsserver.js';
|
|
|
3
3
|
import * as interfaces from '../../../ts_interfaces/index.js';
|
|
4
4
|
import { requireOpsAuth } from '../helpers/auth.js';
|
|
5
5
|
import { CachedEmail } from '../../db/index.js';
|
|
6
|
+
import {
|
|
7
|
+
aggregateEmailLogTrafficInMemory,
|
|
8
|
+
buildEmailLogTraffic,
|
|
9
|
+
EMAIL_LOG_RETENTION_MS,
|
|
10
|
+
normalizeEmailLogSearch,
|
|
11
|
+
selectEmailLogTrafficWindow,
|
|
12
|
+
} from '../../email/helpers.email-log-traffic.js';
|
|
13
|
+
import { evaluateInboundAcceptance } from '../../email/helpers.inbound-security.js';
|
|
6
14
|
|
|
7
15
|
export class EmailOpsHandler {
|
|
8
16
|
constructor(private opsServerRef: OpsServer) {
|
|
@@ -21,12 +29,29 @@ export class EmailOpsHandler {
|
|
|
21
29
|
'getAllEmails',
|
|
22
30
|
async (dataArg) => {
|
|
23
31
|
await requireOpsAuth(this.opsServerRef, dataArg, { scope: 'emails:read' });
|
|
24
|
-
|
|
25
|
-
|
|
32
|
+
return await this.getEmailLogSnapshot(
|
|
33
|
+
dataArg.direction,
|
|
34
|
+
dataArg.limit,
|
|
35
|
+
dataArg.search,
|
|
36
|
+
dataArg.from,
|
|
37
|
+
dataArg.to,
|
|
38
|
+
);
|
|
26
39
|
}
|
|
27
40
|
)
|
|
28
41
|
);
|
|
29
42
|
|
|
43
|
+
viewRouter.addTypedHandler(
|
|
44
|
+
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetEmailSecurityFindings>(
|
|
45
|
+
'getEmailSecurityFindings',
|
|
46
|
+
async (dataArg) => {
|
|
47
|
+
await requireOpsAuth(this.opsServerRef, dataArg, { scope: 'emails:read' });
|
|
48
|
+
return {
|
|
49
|
+
findings: await this.getEmailSecurityFindings(dataArg.limit),
|
|
50
|
+
};
|
|
51
|
+
},
|
|
52
|
+
),
|
|
53
|
+
);
|
|
54
|
+
|
|
30
55
|
// Get Email Detail Handler
|
|
31
56
|
viewRouter.addTypedHandler(
|
|
32
57
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetEmailDetail>(
|
|
@@ -92,10 +117,19 @@ export class EmailOpsHandler {
|
|
|
92
117
|
private async getAllEmails(
|
|
93
118
|
direction?: interfaces.requests.TEmailDirection,
|
|
94
119
|
limitArg?: number,
|
|
120
|
+
searchArg?: string,
|
|
121
|
+
fromArg?: number,
|
|
122
|
+
toArg?: number,
|
|
95
123
|
): Promise<interfaces.requests.IEmail[]> {
|
|
96
124
|
const limit = Math.min(Math.max(Math.floor(limitArg ?? 200), 1), 1000);
|
|
125
|
+
const search = normalizeEmailLogSearch(searchArg);
|
|
126
|
+
const range = this.normalizeEmailLogRange(fromArg, toArg);
|
|
97
127
|
const cachedDocs = this.isCachedEmailStoreAvailable()
|
|
98
|
-
? await CachedEmail.findRecent(limit, {
|
|
128
|
+
? await CachedEmail.findRecent(limit, {
|
|
129
|
+
direction,
|
|
130
|
+
search: search || undefined,
|
|
131
|
+
...(range || {}),
|
|
132
|
+
})
|
|
99
133
|
: [];
|
|
100
134
|
const emails = cachedDocs.map((doc) => this.mapCachedEmailToEmail(doc));
|
|
101
135
|
const representedIds = new Set(cachedDocs.map((doc) => doc.id));
|
|
@@ -110,7 +144,14 @@ export class EmailOpsHandler {
|
|
|
110
144
|
if (cachedId && representedIds.has(cachedId)) {
|
|
111
145
|
continue;
|
|
112
146
|
}
|
|
113
|
-
|
|
147
|
+
const email = this.mapQueueItemToEmail(item);
|
|
148
|
+
const emailTimestamp = new Date(email.timestamp).getTime();
|
|
149
|
+
if (
|
|
150
|
+
this.matchesEmailSearch(email, search)
|
|
151
|
+
&& (!range || (emailTimestamp >= range.from && emailTimestamp <= range.to))
|
|
152
|
+
) {
|
|
153
|
+
emails.push(email);
|
|
154
|
+
}
|
|
114
155
|
}
|
|
115
156
|
}
|
|
116
157
|
|
|
@@ -120,6 +161,116 @@ export class EmailOpsHandler {
|
|
|
120
161
|
return emails.slice(0, limit);
|
|
121
162
|
}
|
|
122
163
|
|
|
164
|
+
private async getEmailLogSnapshot(
|
|
165
|
+
direction?: interfaces.requests.TEmailDirection,
|
|
166
|
+
limitArg?: number,
|
|
167
|
+
searchArg?: string,
|
|
168
|
+
fromArg?: number,
|
|
169
|
+
toArg?: number,
|
|
170
|
+
): Promise<interfaces.requests.IReq_GetAllEmails['response']> {
|
|
171
|
+
const search = normalizeEmailLogSearch(searchArg);
|
|
172
|
+
const now = Date.now();
|
|
173
|
+
const storeAvailable = this.isCachedEmailStoreAvailable();
|
|
174
|
+
const filter = { direction, search: search || undefined };
|
|
175
|
+
const emails = await this.getAllEmails(direction, limitArg, search, fromArg, toArg);
|
|
176
|
+
const trafficContextEmails = !storeAvailable && (fromArg !== undefined || toArg !== undefined)
|
|
177
|
+
? await this.getAllEmails(direction, 1000, search)
|
|
178
|
+
: emails;
|
|
179
|
+
|
|
180
|
+
let oldestMatchAt: number | null = null;
|
|
181
|
+
if (search) {
|
|
182
|
+
oldestMatchAt = storeAvailable
|
|
183
|
+
? await CachedEmail.findOldestAcceptedAt(filter, now - EMAIL_LOG_RETENTION_MS)
|
|
184
|
+
: trafficContextEmails.reduce<number | null>((oldest, email) => {
|
|
185
|
+
const timestamp = new Date(email.timestamp).getTime();
|
|
186
|
+
if (!Number.isFinite(timestamp)) return oldest;
|
|
187
|
+
return oldest === null ? timestamp : Math.min(oldest, timestamp);
|
|
188
|
+
}, null);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const descriptor = selectEmailLogTrafficWindow(search, oldestMatchAt, now);
|
|
192
|
+
const emptyTraffic = buildEmailLogTraffic(descriptor, [], now);
|
|
193
|
+
const sparseTraffic = storeAvailable
|
|
194
|
+
? await CachedEmail.aggregateEmailLogTraffic(
|
|
195
|
+
filter,
|
|
196
|
+
emptyTraffic.windowStart,
|
|
197
|
+
emptyTraffic.windowEnd,
|
|
198
|
+
descriptor.bucketSizeMs,
|
|
199
|
+
)
|
|
200
|
+
: aggregateEmailLogTrafficInMemory(trafficContextEmails, descriptor, now);
|
|
201
|
+
|
|
202
|
+
return {
|
|
203
|
+
emails,
|
|
204
|
+
traffic: buildEmailLogTraffic(descriptor, sparseTraffic, now),
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
private normalizeEmailLogRange(
|
|
209
|
+
fromArg?: number,
|
|
210
|
+
toArg?: number,
|
|
211
|
+
): { from: number; to: number } | undefined {
|
|
212
|
+
if (fromArg === undefined && toArg === undefined) return undefined;
|
|
213
|
+
const now = Date.now();
|
|
214
|
+
if (
|
|
215
|
+
!Number.isSafeInteger(fromArg)
|
|
216
|
+
|| !Number.isSafeInteger(toArg)
|
|
217
|
+
|| (fromArg as number) < now - EMAIL_LOG_RETENTION_MS
|
|
218
|
+
|| (toArg as number) > now + 60_000
|
|
219
|
+
|| (toArg as number) < (fromArg as number)
|
|
220
|
+
) {
|
|
221
|
+
throw new plugins.typedrequest.TypedResponseError(
|
|
222
|
+
'Email log time range is invalid or outside retention.',
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
return { from: fromArg as number, to: toArg as number };
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
private async getEmailSecurityFindings(
|
|
229
|
+
limitArg?: number,
|
|
230
|
+
): Promise<interfaces.requests.IEmailSecurityFinding[]> {
|
|
231
|
+
if (!this.isCachedEmailStoreAvailable()) return [];
|
|
232
|
+
const limit = Math.min(Math.max(Math.floor(limitArg ?? 200), 1), 200);
|
|
233
|
+
const candidates = await CachedEmail.findRecentSecurityCandidates(
|
|
234
|
+
Math.min(limit * 5, 1000),
|
|
235
|
+
Date.now() - EMAIL_LOG_RETENTION_MS,
|
|
236
|
+
);
|
|
237
|
+
const findings: interfaces.requests.IEmailSecurityFinding[] = [];
|
|
238
|
+
for (const candidate of candidates) {
|
|
239
|
+
let securityResults: unknown;
|
|
240
|
+
try {
|
|
241
|
+
securityResults = JSON.parse(candidate.inboundSecurityResults || '');
|
|
242
|
+
} catch {
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
const evaluation = evaluateInboundAcceptance(securityResults);
|
|
246
|
+
if (evaluation.doubts.length === 0) continue;
|
|
247
|
+
findings.push({
|
|
248
|
+
id: candidate.id,
|
|
249
|
+
timestamp: new Date(candidate.acceptedAt || 0).toISOString(),
|
|
250
|
+
from: candidate.from || '',
|
|
251
|
+
to: (candidate.to || []).join(', '),
|
|
252
|
+
subject: candidate.subject || '',
|
|
253
|
+
disposition: candidate.status === 'rejected' || evaluation.dmarcReject
|
|
254
|
+
? 'rejected'
|
|
255
|
+
: 'flagged',
|
|
256
|
+
failures: evaluation.doubts,
|
|
257
|
+
failureSummary: evaluation.doubts.join('; '),
|
|
258
|
+
});
|
|
259
|
+
if (findings.length >= limit) break;
|
|
260
|
+
}
|
|
261
|
+
return findings;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
private matchesEmailSearch(
|
|
265
|
+
emailArg: interfaces.requests.IEmail,
|
|
266
|
+
searchArg: string,
|
|
267
|
+
): boolean {
|
|
268
|
+
if (!searchArg) return true;
|
|
269
|
+
const search = searchArg.toLowerCase();
|
|
270
|
+
return [emailArg.from, emailArg.to, emailArg.subject, emailArg.messageId]
|
|
271
|
+
.some((value) => value.toLowerCase().includes(search));
|
|
272
|
+
}
|
|
273
|
+
|
|
123
274
|
/**
|
|
124
275
|
* Get a single email detail by ID — live queue item first (freshest state),
|
|
125
276
|
* else the durable CachedEmail row (covers inbound + historical mail).
|
|
@@ -149,10 +300,15 @@ export class EmailOpsHandler {
|
|
|
149
300
|
* Map a CachedEmail row to catalog IEmail format
|
|
150
301
|
*/
|
|
151
302
|
private mapCachedEmailToEmail(doc: CachedEmail): interfaces.requests.IEmail {
|
|
303
|
+
const mappedStatus = this.mapStatus(doc.status);
|
|
152
304
|
return {
|
|
153
305
|
id: doc.id,
|
|
154
306
|
direction: doc.direction,
|
|
155
|
-
status:
|
|
307
|
+
status: doc.direction === 'inbound'
|
|
308
|
+
&& mappedStatus === 'accepted'
|
|
309
|
+
&& doc.inboundSecurityDisposition === 'flagged'
|
|
310
|
+
? 'flagged'
|
|
311
|
+
: mappedStatus,
|
|
156
312
|
from: doc.from || '',
|
|
157
313
|
to: doc.to?.[0] || '',
|
|
158
314
|
subject: doc.subject || '',
|
|
@@ -183,9 +339,13 @@ export class EmailOpsHandler {
|
|
|
183
339
|
// Display SmartMTA's true retry schedule — doc.nextAttempt is the spool's
|
|
184
340
|
// recovery lease (retry time + 30min grace), not the user-facing countdown.
|
|
185
341
|
const nextAttemptMs = doc.smartMtaNextAttempt ? new Date(doc.smartMtaNextAttempt).getTime() : NaN;
|
|
342
|
+
const outboundAuthentication = doc.direction === 'outbound'
|
|
343
|
+
? this.getLatestOutboundAuthentication(doc.smtpTransactions || [])
|
|
344
|
+
: undefined;
|
|
186
345
|
return {
|
|
187
346
|
...base,
|
|
188
347
|
smtpTransactions: structuredClone(doc.smtpTransactions || []),
|
|
348
|
+
...(outboundAuthentication ? { outboundAuthentication } : {}),
|
|
189
349
|
...(base.status === 'deferred' && Number.isFinite(nextAttemptMs) && nextAttemptMs > 0
|
|
190
350
|
? {
|
|
191
351
|
nextAttemptAt: nextAttemptMs,
|
|
@@ -206,7 +366,9 @@ export class EmailOpsHandler {
|
|
|
206
366
|
authMethod: '',
|
|
207
367
|
authUser: this.extractSessionUser(session),
|
|
208
368
|
},
|
|
209
|
-
authenticationResults:
|
|
369
|
+
authenticationResults: outboundAuthentication
|
|
370
|
+
? this.mapOutboundAuthenticationResults(outboundAuthentication)
|
|
371
|
+
: this.mapInboundSecurityResults(doc.inboundSecurityResults),
|
|
210
372
|
rejectionReason: doc.status === 'failed' ? doc.lastError : undefined,
|
|
211
373
|
bounceMessage: doc.status === 'failed' ? doc.lastError : undefined,
|
|
212
374
|
headers,
|
|
@@ -337,6 +499,7 @@ export class EmailOpsHandler {
|
|
|
337
499
|
const base = this.mapQueueItemToEmail(item);
|
|
338
500
|
const processingResult = item.processingResult;
|
|
339
501
|
|
|
502
|
+
const outboundAuthentication = this.getLatestOutboundAuthentication(item.smtpTransactions || []);
|
|
340
503
|
let toList: string[] = [];
|
|
341
504
|
let cc: string[] = [];
|
|
342
505
|
let headers: Record<string, string> = {};
|
|
@@ -359,6 +522,7 @@ export class EmailOpsHandler {
|
|
|
359
522
|
...base,
|
|
360
523
|
smtpTransactions: structuredClone(item.smtpTransactions || []),
|
|
361
524
|
toList,
|
|
525
|
+
...(outboundAuthentication ? { outboundAuthentication } : {}),
|
|
362
526
|
cc,
|
|
363
527
|
smtpLog: this.flattenSmtpTransactions(item.smtpTransactions || []),
|
|
364
528
|
connectionInfo: {
|
|
@@ -372,14 +536,7 @@ export class EmailOpsHandler {
|
|
|
372
536
|
authMethod: '',
|
|
373
537
|
authUser: '',
|
|
374
538
|
},
|
|
375
|
-
authenticationResults:
|
|
376
|
-
spf: 'none',
|
|
377
|
-
spfDomain: '',
|
|
378
|
-
dkim: 'none',
|
|
379
|
-
dkimDomain: '',
|
|
380
|
-
dmarc: 'none',
|
|
381
|
-
dmarcPolicy: '',
|
|
382
|
-
},
|
|
539
|
+
authenticationResults: this.mapOutboundAuthenticationResults(outboundAuthentication),
|
|
383
540
|
rejectionReason: item.status === 'failed' ? item.lastError : undefined,
|
|
384
541
|
bounceMessage: item.status === 'failed' ? item.lastError : undefined,
|
|
385
542
|
headers,
|
|
@@ -387,6 +544,47 @@ export class EmailOpsHandler {
|
|
|
387
544
|
};
|
|
388
545
|
}
|
|
389
546
|
|
|
547
|
+
private getLatestOutboundAuthentication(
|
|
548
|
+
transactionsArg: Array<{ outboundAuthentication?: interfaces.requests.IOutboundAuthenticationCheck }>,
|
|
549
|
+
): interfaces.requests.IOutboundAuthenticationCheck | undefined {
|
|
550
|
+
for (let index = transactionsArg.length - 1; index >= 0; index--) {
|
|
551
|
+
const authentication = transactionsArg[index]?.outboundAuthentication;
|
|
552
|
+
if (!authentication) continue;
|
|
553
|
+
return structuredClone(authentication);
|
|
554
|
+
}
|
|
555
|
+
return undefined;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
private mapOutboundAuthenticationResults(
|
|
559
|
+
authenticationArg?: interfaces.requests.IOutboundAuthenticationCheck,
|
|
560
|
+
): interfaces.requests.IEmailDetail['authenticationResults'] {
|
|
561
|
+
const noneResult: interfaces.requests.IEmailDetail['authenticationResults'] = {
|
|
562
|
+
spf: 'none',
|
|
563
|
+
spfDomain: '',
|
|
564
|
+
dkim: 'none',
|
|
565
|
+
dkimDomain: '',
|
|
566
|
+
dmarc: 'none',
|
|
567
|
+
dmarcPolicy: '',
|
|
568
|
+
};
|
|
569
|
+
if (authenticationArg?.state !== 'completed') return noneResult;
|
|
570
|
+
const dkimSignatures = (authenticationArg.dkim || [])
|
|
571
|
+
.filter((signature) => signature.status.toLowerCase() !== 'none');
|
|
572
|
+
const validDkim = dkimSignatures.find((signature) => signature.is_valid);
|
|
573
|
+
const spfValue = (authenticationArg.spf?.result || '').toLowerCase();
|
|
574
|
+
const spf = (['pass', 'fail', 'softfail', 'neutral'] as string[]).includes(spfValue)
|
|
575
|
+
? spfValue as interfaces.requests.IAuthenticationResults['spf']
|
|
576
|
+
: 'none';
|
|
577
|
+
return {
|
|
578
|
+
spf,
|
|
579
|
+
spfDomain: authenticationArg.spf?.domain || '',
|
|
580
|
+
dkim: validDkim ? 'pass' : dkimSignatures.length > 0 ? 'fail' : 'none',
|
|
581
|
+
dkimDomain: validDkim?.domain || dkimSignatures[0]?.domain || '',
|
|
582
|
+
dmarc: authenticationArg.dmarc ? (authenticationArg.dmarc.passed ? 'pass' : 'fail') : 'none',
|
|
583
|
+
dmarcPolicy: authenticationArg.dmarc?.policy || '',
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
|
|
390
588
|
/**
|
|
391
589
|
* Map queue status to catalog TEmailStatus
|
|
392
590
|
*/
|
|
@@ -413,7 +611,8 @@ export class EmailOpsHandler {
|
|
|
413
611
|
};
|
|
414
612
|
const spfResult = (parsed.spf?.result || '').toLowerCase();
|
|
415
613
|
const spf = (['pass', 'fail', 'softfail', 'neutral'] as const).find((value) => value === spfResult) || 'none';
|
|
416
|
-
const dkimSignatures = Array.isArray(parsed.dkim) ? parsed.dkim : []
|
|
614
|
+
const dkimSignatures = (Array.isArray(parsed.dkim) ? parsed.dkim : [])
|
|
615
|
+
.filter((signature) => String(signature?.status || '').toLowerCase() !== 'none');
|
|
417
616
|
const validSignature = dkimSignatures.find((signature) => signature?.is_valid);
|
|
418
617
|
const dkim = validSignature ? 'pass' : dkimSignatures.length > 0 ? 'fail' : 'none';
|
|
419
618
|
const dmarc = parsed.dmarc ? (parsed.dmarc.passed ? 'pass' : 'fail') : 'none';
|
|
@@ -370,6 +370,20 @@ export class TunnelManager {
|
|
|
370
370
|
return { ready: false, eligibleEdgeCount: 0, reason };
|
|
371
371
|
}
|
|
372
372
|
|
|
373
|
+
/**
|
|
374
|
+
* Delegate source-bound connection evidence to the owned RemoteIngress hub.
|
|
375
|
+
*/
|
|
376
|
+
public async waitForEgressConnectionEvidence(
|
|
377
|
+
proxyId: string,
|
|
378
|
+
edgeId: string,
|
|
379
|
+
timeoutMs?: number,
|
|
380
|
+
): Promise<plugins.remoteingress.IRemoteIngressHubEgressConnectionEvent | null> {
|
|
381
|
+
if (this.stopped) {
|
|
382
|
+
return null;
|
|
383
|
+
}
|
|
384
|
+
return await this.hub.waitForEgressConnectionEvidence(proxyId, edgeId, timeoutMs);
|
|
385
|
+
}
|
|
386
|
+
|
|
373
387
|
public async stopEgressTcpProxy(proxyId: string): Promise<boolean> {
|
|
374
388
|
if (this.stopped) {
|
|
375
389
|
return false;
|