@serve.zone/dcrouter 18.1.0 → 18.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deno.json +1 -1
- package/dist_serve/bundle.js +2072 -1624
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.dcrouter.d.ts +64 -2
- package/dist_ts/classes.dcrouter.js +187 -6
- package/dist_ts/db/documents/classes.authentication-event.doc.d.ts +25 -0
- package/dist_ts/db/documents/classes.authentication-event.doc.js +273 -0
- package/dist_ts/db/documents/classes.email-traffic-bucket.doc.d.ts +25 -0
- package/dist_ts/db/documents/classes.email-traffic-bucket.doc.js +200 -0
- package/dist_ts/db/documents/index.d.ts +2 -0
- package/dist_ts/db/documents/index.js +3 -1
- package/dist_ts/email/classes.accepted-email-spool.d.ts +67 -0
- package/dist_ts/email/classes.accepted-email-spool.js +245 -17
- package/dist_ts/email/classes.workapp-mail-manager.js +5 -2
- package/dist_ts/monitoring/classes.metricsmanager.d.ts +19 -0
- package/dist_ts/monitoring/classes.metricsmanager.js +240 -44
- package/dist_ts/opsserver/handlers/admin.handler.d.ts +1 -0
- package/dist_ts/opsserver/handlers/admin.handler.js +61 -14
- package/dist_ts/opsserver/handlers/security.handler.js +21 -3
- package/dist_ts/opsserver/handlers/stats.handler.js +27 -10
- package/dist_ts/security/classes.authentication-event-manager.d.ts +35 -0
- package/dist_ts/security/classes.authentication-event-manager.js +185 -0
- package/dist_ts/security/index.d.ts +1 -0
- package/dist_ts/security/index.js +2 -1
- package/dist_ts_interfaces/data/stats.d.ts +16 -0
- package/dist_ts_migrations/index.js +74 -15
- package/dist_ts_web/00_commitinfo_data.js +1 -1
- package/dist_ts_web/elements/overview/ops-view-overview.js +3 -2
- package/dist_ts_web/elements/security/ops-view-security-authentication.js +20 -13
- package/dist_ts_web/elements/security/ops-view-security-overview.js +8 -11
- package/package.json +3 -3
- package/readme.md +24 -0
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.dcrouter.ts +230 -6
- package/ts/db/documents/classes.authentication-event.doc.ts +248 -0
- package/ts/db/documents/classes.email-traffic-bucket.doc.ts +161 -0
- package/ts/db/documents/index.ts +2 -0
- package/ts/email/classes.accepted-email-spool.ts +319 -21
- package/ts/email/classes.workapp-mail-manager.ts +4 -1
- package/ts/monitoring/classes.metricsmanager.ts +287 -50
- package/ts/opsserver/handlers/admin.handler.ts +74 -15
- package/ts/opsserver/handlers/security.handler.ts +23 -2
- package/ts/opsserver/handlers/stats.handler.ts +29 -9
- package/ts/security/classes.authentication-event-manager.ts +221 -0
- package/ts/security/index.ts +1 -0
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/elements/overview/ops-view-overview.ts +2 -1
- package/ts_web/elements/security/ops-view-security-authentication.ts +20 -13
- package/ts_web/elements/security/ops-view-security-overview.ts +7 -12
|
@@ -4,7 +4,15 @@ import { MetricsCache } from './classes.metricscache.js';
|
|
|
4
4
|
import { SecurityLogger, SecurityEventType } from '../security/classes.securitylogger.js';
|
|
5
5
|
import { logger } from '../logger.js';
|
|
6
6
|
import type { IAsnActivity, IProtocolConnectionHistoryPoint } from '../../ts_interfaces/data/stats.js';
|
|
7
|
-
|
|
7
|
+
import {
|
|
8
|
+
EmailTrafficBucketDoc,
|
|
9
|
+
type IEmailTrafficBucketSnapshot,
|
|
10
|
+
} from '../db/documents/classes.email-traffic-bucket.doc.js';
|
|
11
|
+
|
|
12
|
+
const EMAIL_TRAFFIC_WINDOW_MINUTES = 24 * 60;
|
|
13
|
+
const EMAIL_TRAFFIC_FLUSH_INTERVAL_MS = 5_000;
|
|
14
|
+
const EMAIL_TRAFFIC_PRUNE_INTERVAL_MS = 60 * 60 * 1000;
|
|
15
|
+
const AUTHENTICATION_EVENT_RETENTION_MS = 90 * 24 * 60 * 60 * 1000;
|
|
8
16
|
const NETWORK_STATS_CACHE_TTL_MS = 3_000;
|
|
9
17
|
const ASN_INTELLIGENCE_CACHE_TTL_MS = 60_000;
|
|
10
18
|
const ACTIVE_CONNECTION_SNAPSHOT_CACHE_TTL_MS = 3_000;
|
|
@@ -20,6 +28,12 @@ export class MetricsManager {
|
|
|
20
28
|
private dcRouter: DcRouter;
|
|
21
29
|
private resetInterval?: NodeJS.Timeout;
|
|
22
30
|
private observedIpSamplerInterval?: NodeJS.Timeout;
|
|
31
|
+
private emailTrafficFlushInterval?: NodeJS.Timeout;
|
|
32
|
+
private persistedMetricsPruneInterval?: NodeJS.Timeout;
|
|
33
|
+
private emailTrafficFlushChain: Promise<void> = Promise.resolve();
|
|
34
|
+
private persistedMetricsFlushInFlight?: Promise<void>;
|
|
35
|
+
private persistedMetricsPruneInFlight?: Promise<void>;
|
|
36
|
+
private emailDirtyBuckets = new Map<number, number>();
|
|
23
37
|
private metricsCache: MetricsCache;
|
|
24
38
|
|
|
25
39
|
// Constants
|
|
@@ -32,7 +46,7 @@ export class MetricsManager {
|
|
|
32
46
|
failedToday: 0,
|
|
33
47
|
bouncedToday: 0,
|
|
34
48
|
queueSize: 0,
|
|
35
|
-
lastResetDate: new Date().
|
|
49
|
+
lastResetDate: new Date().toISOString().slice(0, 10),
|
|
36
50
|
deliveryTimes: [] as number[], // Track delivery times in ms
|
|
37
51
|
recipients: new Map<string, number>(), // Track email count by recipient
|
|
38
52
|
recentActivity: [] as Array<{ timestamp: number; type: string; details: string }>,
|
|
@@ -54,7 +68,9 @@ export class MetricsManager {
|
|
|
54
68
|
};
|
|
55
69
|
|
|
56
70
|
// Per-minute time-series buckets for charts
|
|
57
|
-
private emailMinuteBuckets = new Map<number, {
|
|
71
|
+
private emailMinuteBuckets = new Map<number, {
|
|
72
|
+
sent: number; received: number; failed: number; revision: number;
|
|
73
|
+
}>();
|
|
58
74
|
private dnsMinuteBuckets = new Map<number, { queries: number }>();
|
|
59
75
|
|
|
60
76
|
// Track security-specific metrics
|
|
@@ -84,23 +100,18 @@ export class MetricsManager {
|
|
|
84
100
|
}
|
|
85
101
|
|
|
86
102
|
public async start(): Promise<void> {
|
|
103
|
+
if (this.canPersistEmailTraffic()) {
|
|
104
|
+
await this.loadEmailTrafficBuckets();
|
|
105
|
+
await this.prunePersistedMetrics();
|
|
106
|
+
}
|
|
107
|
+
|
|
87
108
|
// Start SmartMetrics collection
|
|
88
109
|
this.smartMetrics.start();
|
|
89
110
|
|
|
90
|
-
//
|
|
111
|
+
// Maintain UTC daily ancillary counters and bounded in-memory windows.
|
|
91
112
|
this.resetInterval = setInterval(() => {
|
|
92
113
|
const currentDate = new Date().toDateString();
|
|
93
|
-
|
|
94
|
-
if (currentDate !== this.emailMetrics.lastResetDate) {
|
|
95
|
-
this.emailMetrics.sentToday = 0;
|
|
96
|
-
this.emailMetrics.receivedToday = 0;
|
|
97
|
-
this.emailMetrics.failedToday = 0;
|
|
98
|
-
this.emailMetrics.bouncedToday = 0;
|
|
99
|
-
this.emailMetrics.deliveryTimes = [];
|
|
100
|
-
this.emailMetrics.recipients.clear();
|
|
101
|
-
this.emailMetrics.recentActivity = [];
|
|
102
|
-
this.emailMetrics.lastResetDate = currentDate;
|
|
103
|
-
}
|
|
114
|
+
this.ensureEmailDailyState();
|
|
104
115
|
|
|
105
116
|
if (currentDate !== this.dnsMetrics.lastResetDate) {
|
|
106
117
|
this.dnsMetrics.totalQueries = 0;
|
|
@@ -128,17 +139,33 @@ export class MetricsManager {
|
|
|
128
139
|
// Prune old time-series buckets every minute (don't wait for lazy query)
|
|
129
140
|
this.pruneOldBuckets();
|
|
130
141
|
}, 60000); // Check every minute
|
|
142
|
+
this.resetInterval.unref();
|
|
131
143
|
|
|
132
144
|
this.observedIpSamplerInterval = setInterval(() => {
|
|
133
145
|
this.sampleObservedIps();
|
|
134
146
|
}, OBSERVED_IP_SAMPLE_INTERVAL_MS);
|
|
135
147
|
this.observedIpSamplerInterval.unref();
|
|
148
|
+
|
|
149
|
+
if (this.canPersistEmailTraffic()) {
|
|
150
|
+
this.emailTrafficFlushInterval = setInterval(() => {
|
|
151
|
+
void this.flushPersistedMetrics().catch((error) => {
|
|
152
|
+
logger.log('warn', `Metrics persistence flush failed: ${(error as Error).message}`);
|
|
153
|
+
});
|
|
154
|
+
}, EMAIL_TRAFFIC_FLUSH_INTERVAL_MS);
|
|
155
|
+
this.emailTrafficFlushInterval.unref();
|
|
156
|
+
|
|
157
|
+
this.persistedMetricsPruneInterval = setInterval(() => {
|
|
158
|
+
void this.prunePersistedMetrics().catch((error) => {
|
|
159
|
+
logger.log('warn', `Metrics persistence pruning failed: ${(error as Error).message}`);
|
|
160
|
+
});
|
|
161
|
+
}, EMAIL_TRAFFIC_PRUNE_INTERVAL_MS);
|
|
162
|
+
this.persistedMetricsPruneInterval.unref();
|
|
163
|
+
}
|
|
136
164
|
|
|
137
165
|
logger.log('info', 'MetricsManager started');
|
|
138
166
|
}
|
|
139
167
|
|
|
140
168
|
public async stop(): Promise<void> {
|
|
141
|
-
// Clear the reset interval
|
|
142
169
|
if (this.resetInterval) {
|
|
143
170
|
clearInterval(this.resetInterval);
|
|
144
171
|
this.resetInterval = undefined;
|
|
@@ -147,12 +174,60 @@ export class MetricsManager {
|
|
|
147
174
|
clearInterval(this.observedIpSamplerInterval);
|
|
148
175
|
this.observedIpSamplerInterval = undefined;
|
|
149
176
|
}
|
|
177
|
+
if (this.emailTrafficFlushInterval) {
|
|
178
|
+
clearInterval(this.emailTrafficFlushInterval);
|
|
179
|
+
this.emailTrafficFlushInterval = undefined;
|
|
180
|
+
}
|
|
181
|
+
if (this.persistedMetricsPruneInterval) {
|
|
182
|
+
clearInterval(this.persistedMetricsPruneInterval);
|
|
183
|
+
this.persistedMetricsPruneInterval = undefined;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
if (this.persistedMetricsPruneInFlight) {
|
|
187
|
+
try {
|
|
188
|
+
await this.persistedMetricsPruneInFlight;
|
|
189
|
+
} catch (error) {
|
|
190
|
+
logger.log('warn', `Final metrics pruning wait failed: ${(error as Error).message}`);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
let finalFlushError: Error | undefined;
|
|
195
|
+
if (this.canPersistEmailTraffic()) {
|
|
196
|
+
const activeFlush = this.persistedMetricsFlushInFlight;
|
|
197
|
+
if (activeFlush) {
|
|
198
|
+
try {
|
|
199
|
+
await activeFlush;
|
|
200
|
+
} catch (error) {
|
|
201
|
+
logger.log(
|
|
202
|
+
'warn',
|
|
203
|
+
`Active metrics persistence flush failed before final drain: ${(error as Error).message}`,
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// OpsServer and EmailServer have stopped by this point. Run a fresh,
|
|
209
|
+
// non-coalesced pass so events added after an earlier flush snapshot are
|
|
210
|
+
// included in the final durable drain.
|
|
211
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
212
|
+
try {
|
|
213
|
+
await this.flushPersistedMetricsUnlocked();
|
|
214
|
+
finalFlushError = undefined;
|
|
215
|
+
break;
|
|
216
|
+
} catch (error) {
|
|
217
|
+
finalFlushError = error as Error;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
150
221
|
|
|
151
222
|
this.smartMetrics.stop();
|
|
152
223
|
|
|
153
|
-
|
|
224
|
+
if (finalFlushError) {
|
|
225
|
+
throw new Error(`Final metrics persistence flush failed: ${finalFlushError.message}`);
|
|
226
|
+
}
|
|
227
|
+
|
|
154
228
|
this.metricsCache.clear();
|
|
155
229
|
this.emailMinuteBuckets.clear();
|
|
230
|
+
this.emailDirtyBuckets.clear();
|
|
156
231
|
this.dnsMinuteBuckets.clear();
|
|
157
232
|
|
|
158
233
|
logger.log('info', 'MetricsManager stopped');
|
|
@@ -199,30 +274,31 @@ export class MetricsManager {
|
|
|
199
274
|
// Get email metrics
|
|
200
275
|
public async getEmailStats() {
|
|
201
276
|
return this.metricsCache.get('emailStats', () => {
|
|
202
|
-
|
|
277
|
+
this.ensureEmailDailyState();
|
|
278
|
+
const totals = this.getUtcTodayEmailTotals();
|
|
279
|
+
const deliveryAttempts = totals.sent + totals.failed;
|
|
203
280
|
const avgDeliveryTime = this.emailMetrics.deliveryTimes.length > 0
|
|
204
281
|
? this.emailMetrics.deliveryTimes.reduce((a, b) => a + b, 0) / this.emailMetrics.deliveryTimes.length
|
|
205
282
|
: 0;
|
|
206
|
-
|
|
207
|
-
// Get top recipients
|
|
283
|
+
|
|
208
284
|
const topRecipients = Array.from(this.emailMetrics.recipients.entries())
|
|
209
285
|
.sort((a, b) => b[1] - a[1])
|
|
210
286
|
.slice(0, 10)
|
|
211
287
|
.map(([email, count]) => ({ email, count }));
|
|
212
|
-
|
|
213
|
-
// Get recent activity (last 50 entries)
|
|
288
|
+
|
|
214
289
|
const recentActivity = this.emailMetrics.recentActivity.slice(-50);
|
|
215
|
-
|
|
290
|
+
|
|
216
291
|
return {
|
|
217
|
-
sentToday:
|
|
218
|
-
receivedToday:
|
|
219
|
-
failedToday:
|
|
220
|
-
|
|
221
|
-
|
|
292
|
+
sentToday: totals.sent,
|
|
293
|
+
receivedToday: totals.received,
|
|
294
|
+
failedToday: totals.failed,
|
|
295
|
+
bouncedToday: this.emailMetrics.bouncedToday,
|
|
296
|
+
bounceRate: deliveryAttempts > 0
|
|
297
|
+
? Math.min(1, Math.max(0, this.emailMetrics.bouncedToday / deliveryAttempts))
|
|
222
298
|
: 0,
|
|
223
|
-
deliveryRate:
|
|
224
|
-
? ((
|
|
225
|
-
:
|
|
299
|
+
deliveryRate: deliveryAttempts > 0
|
|
300
|
+
? Math.min(1, Math.max(0, totals.sent / deliveryAttempts))
|
|
301
|
+
: 1,
|
|
226
302
|
queueSize: this.emailMetrics.queueSize,
|
|
227
303
|
averageDeliveryTime: Math.round(avgDeliveryTime),
|
|
228
304
|
topRecipients,
|
|
@@ -278,8 +354,13 @@ export class MetricsManager {
|
|
|
278
354
|
this.securityMetrics.spamDetected = summary.byType[SecurityEventType.SPAM] || 0;
|
|
279
355
|
this.securityMetrics.malwareDetected = summary.byType[SecurityEventType.MALWARE] || 0;
|
|
280
356
|
this.securityMetrics.phishingDetected = summary.byType[SecurityEventType.DMARC] || 0; // phishing via DMARC
|
|
281
|
-
this.securityMetrics.authFailures =
|
|
282
|
-
|
|
357
|
+
this.securityMetrics.authFailures = securityLogger
|
|
358
|
+
.getRecentEvents(10_000, {
|
|
359
|
+
type: SecurityEventType.AUTHENTICATION,
|
|
360
|
+
fromTimestamp: Date.now() - 86_400_000,
|
|
361
|
+
})
|
|
362
|
+
.filter((event) => event.success === false)
|
|
363
|
+
.length;
|
|
283
364
|
this.securityMetrics.blockedIPs =
|
|
284
365
|
(summary.byType[SecurityEventType.IP_REPUTATION] || 0) +
|
|
285
366
|
(summary.byType[SecurityEventType.REJECTED_CONNECTION] || 0);
|
|
@@ -377,7 +458,7 @@ export class MetricsManager {
|
|
|
377
458
|
|
|
378
459
|
// Email event tracking methods
|
|
379
460
|
public trackEmailSent(recipient?: string, deliveryTimeMs?: number): void {
|
|
380
|
-
this.
|
|
461
|
+
this.ensureEmailDailyState();
|
|
381
462
|
this.incrementEmailBucket('sent');
|
|
382
463
|
|
|
383
464
|
if (recipient) {
|
|
@@ -414,7 +495,7 @@ export class MetricsManager {
|
|
|
414
495
|
}
|
|
415
496
|
|
|
416
497
|
public trackEmailReceived(sender?: string): void {
|
|
417
|
-
this.
|
|
498
|
+
this.ensureEmailDailyState();
|
|
418
499
|
this.incrementEmailBucket('received');
|
|
419
500
|
|
|
420
501
|
this.emailMetrics.recentActivity.push({
|
|
@@ -430,7 +511,7 @@ export class MetricsManager {
|
|
|
430
511
|
}
|
|
431
512
|
|
|
432
513
|
public trackEmailFailed(recipient?: string, reason?: string): void {
|
|
433
|
-
this.
|
|
514
|
+
this.ensureEmailDailyState();
|
|
434
515
|
this.incrementEmailBucket('failed');
|
|
435
516
|
|
|
436
517
|
this.emailMetrics.recentActivity.push({
|
|
@@ -446,6 +527,7 @@ export class MetricsManager {
|
|
|
446
527
|
}
|
|
447
528
|
|
|
448
529
|
public trackEmailBounced(recipient?: string): void {
|
|
530
|
+
this.ensureEmailDailyState();
|
|
449
531
|
this.emailMetrics.bouncedToday++;
|
|
450
532
|
|
|
451
533
|
this.emailMetrics.recentActivity.push({
|
|
@@ -1087,14 +1169,164 @@ export class MetricsManager {
|
|
|
1087
1169
|
return Math.floor(ts / 60000) * 60000;
|
|
1088
1170
|
}
|
|
1089
1171
|
|
|
1172
|
+
private static firstMinuteForHours(hoursArg: number, nowArg = Date.now()): number {
|
|
1173
|
+
const pointCount = Math.max(1, Math.floor(hoursArg * 60));
|
|
1174
|
+
return MetricsManager.minuteKey(nowArg) - (pointCount - 1) * 60_000;
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
private isEmailTrafficPersistenceConfigured(): boolean {
|
|
1178
|
+
return this.dcRouter.options.dbConfig?.enabled !== false;
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
private canPersistEmailTraffic(): boolean {
|
|
1182
|
+
return this.isEmailTrafficPersistenceConfigured()
|
|
1183
|
+
&& Boolean(this.dcRouter.dcRouterDb?.isReady());
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
private ensureEmailDailyState(nowArg = Date.now()): void {
|
|
1187
|
+
const currentUtcDate = new Date(nowArg).toISOString().slice(0, 10);
|
|
1188
|
+
if (currentUtcDate === this.emailMetrics.lastResetDate) return;
|
|
1189
|
+
|
|
1190
|
+
this.emailMetrics.sentToday = 0;
|
|
1191
|
+
this.emailMetrics.receivedToday = 0;
|
|
1192
|
+
this.emailMetrics.failedToday = 0;
|
|
1193
|
+
this.emailMetrics.bouncedToday = 0;
|
|
1194
|
+
this.emailMetrics.deliveryTimes = [];
|
|
1195
|
+
this.emailMetrics.recipients.clear();
|
|
1196
|
+
this.emailMetrics.recentActivity = [];
|
|
1197
|
+
this.emailMetrics.lastResetDate = currentUtcDate;
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
private getUtcTodayEmailTotals(nowArg = Date.now()): {
|
|
1201
|
+
sent: number;
|
|
1202
|
+
received: number;
|
|
1203
|
+
failed: number;
|
|
1204
|
+
} {
|
|
1205
|
+
const now = new Date(nowArg);
|
|
1206
|
+
const utcMidnight = Date.UTC(
|
|
1207
|
+
now.getUTCFullYear(),
|
|
1208
|
+
now.getUTCMonth(),
|
|
1209
|
+
now.getUTCDate(),
|
|
1210
|
+
);
|
|
1211
|
+
const totals = { sent: 0, received: 0, failed: 0 };
|
|
1212
|
+
for (const [bucketStart, bucket] of this.emailMinuteBuckets) {
|
|
1213
|
+
if (bucketStart < utcMidnight || bucketStart > nowArg) continue;
|
|
1214
|
+
totals.sent += bucket.sent;
|
|
1215
|
+
totals.received += bucket.received;
|
|
1216
|
+
totals.failed += bucket.failed;
|
|
1217
|
+
}
|
|
1218
|
+
return totals;
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
private async loadEmailTrafficBuckets(): Promise<void> {
|
|
1222
|
+
const snapshots = await EmailTrafficBucketDoc.loadSince(
|
|
1223
|
+
MetricsManager.firstMinuteForHours(24),
|
|
1224
|
+
);
|
|
1225
|
+
for (const snapshot of snapshots) {
|
|
1226
|
+
this.emailMinuteBuckets.set(snapshot.bucketStart, {
|
|
1227
|
+
sent: snapshot.sent,
|
|
1228
|
+
received: snapshot.received,
|
|
1229
|
+
failed: snapshot.failed,
|
|
1230
|
+
revision: 0,
|
|
1231
|
+
});
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
private flushPersistedMetrics(): Promise<void> {
|
|
1236
|
+
if (this.persistedMetricsFlushInFlight) return this.persistedMetricsFlushInFlight;
|
|
1237
|
+
const run = this.flushPersistedMetricsUnlocked();
|
|
1238
|
+
this.persistedMetricsFlushInFlight = run;
|
|
1239
|
+
run.then(
|
|
1240
|
+
() => { if (this.persistedMetricsFlushInFlight === run) this.persistedMetricsFlushInFlight = undefined; },
|
|
1241
|
+
() => { if (this.persistedMetricsFlushInFlight === run) this.persistedMetricsFlushInFlight = undefined; },
|
|
1242
|
+
);
|
|
1243
|
+
return run;
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
private async flushPersistedMetricsUnlocked(): Promise<void> {
|
|
1247
|
+
const results = await Promise.allSettled([
|
|
1248
|
+
this.flushEmailTrafficBuckets(),
|
|
1249
|
+
this.dcRouter.authenticationEventManager.flushBuffered(),
|
|
1250
|
+
]);
|
|
1251
|
+
const failure = results.find((result) => result.status === 'rejected');
|
|
1252
|
+
if (failure?.status === 'rejected') {
|
|
1253
|
+
throw failure.reason;
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
private flushEmailTrafficBuckets(): Promise<void> {
|
|
1258
|
+
const run = this.emailTrafficFlushChain.then(
|
|
1259
|
+
() => this.flushEmailTrafficBucketsUnlocked(),
|
|
1260
|
+
() => this.flushEmailTrafficBucketsUnlocked(),
|
|
1261
|
+
);
|
|
1262
|
+
this.emailTrafficFlushChain = run.then(() => undefined, () => undefined);
|
|
1263
|
+
return run;
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
private async flushEmailTrafficBucketsUnlocked(): Promise<void> {
|
|
1267
|
+
if (!this.canPersistEmailTraffic() || this.emailDirtyBuckets.size === 0) return;
|
|
1268
|
+
|
|
1269
|
+
const snapshots: Array<IEmailTrafficBucketSnapshot & { revision: number }> = [];
|
|
1270
|
+
for (const [bucketStart, revision] of this.emailDirtyBuckets) {
|
|
1271
|
+
const bucket = this.emailMinuteBuckets.get(bucketStart);
|
|
1272
|
+
if (!bucket) continue;
|
|
1273
|
+
snapshots.push({
|
|
1274
|
+
bucketStart,
|
|
1275
|
+
sent: bucket.sent,
|
|
1276
|
+
received: bucket.received,
|
|
1277
|
+
failed: bucket.failed,
|
|
1278
|
+
revision,
|
|
1279
|
+
});
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1282
|
+
for (let offset = 0; offset < snapshots.length; offset += 500) {
|
|
1283
|
+
const batch = snapshots.slice(offset, offset + 500);
|
|
1284
|
+
await EmailTrafficBucketDoc.persistAbsolute(batch);
|
|
1285
|
+
for (const snapshot of batch) {
|
|
1286
|
+
const current = this.emailMinuteBuckets.get(snapshot.bucketStart);
|
|
1287
|
+
if (
|
|
1288
|
+
current?.revision === snapshot.revision
|
|
1289
|
+
&& this.emailDirtyBuckets.get(snapshot.bucketStart) === snapshot.revision
|
|
1290
|
+
) {
|
|
1291
|
+
this.emailDirtyBuckets.delete(snapshot.bucketStart);
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
private prunePersistedMetrics(): Promise<void> {
|
|
1298
|
+
if (this.persistedMetricsPruneInFlight) return this.persistedMetricsPruneInFlight;
|
|
1299
|
+
const run = this.prunePersistedMetricsUnlocked();
|
|
1300
|
+
this.persistedMetricsPruneInFlight = run;
|
|
1301
|
+
run.then(
|
|
1302
|
+
() => { if (this.persistedMetricsPruneInFlight === run) this.persistedMetricsPruneInFlight = undefined; },
|
|
1303
|
+
() => { if (this.persistedMetricsPruneInFlight === run) this.persistedMetricsPruneInFlight = undefined; },
|
|
1304
|
+
);
|
|
1305
|
+
return run;
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
private async prunePersistedMetricsUnlocked(): Promise<void> {
|
|
1309
|
+
if (!this.canPersistEmailTraffic()) return;
|
|
1310
|
+
await Promise.all([
|
|
1311
|
+
EmailTrafficBucketDoc.pruneBefore(MetricsManager.firstMinuteForHours(24)),
|
|
1312
|
+
this.dcRouter.authenticationEventManager.pruneBefore(
|
|
1313
|
+
Date.now() - AUTHENTICATION_EVENT_RETENTION_MS,
|
|
1314
|
+
),
|
|
1315
|
+
]);
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1090
1318
|
private incrementEmailBucket(field: 'sent' | 'received' | 'failed'): void {
|
|
1091
1319
|
const key = MetricsManager.minuteKey();
|
|
1092
1320
|
let bucket = this.emailMinuteBuckets.get(key);
|
|
1093
1321
|
if (!bucket) {
|
|
1094
|
-
bucket = { sent: 0, received: 0, failed: 0 };
|
|
1322
|
+
bucket = { sent: 0, received: 0, failed: 0, revision: 0 };
|
|
1095
1323
|
this.emailMinuteBuckets.set(key, bucket);
|
|
1096
1324
|
}
|
|
1097
1325
|
bucket[field]++;
|
|
1326
|
+
bucket.revision++;
|
|
1327
|
+
if (this.isEmailTrafficPersistenceConfigured()) {
|
|
1328
|
+
this.emailDirtyBuckets.set(key, bucket.revision);
|
|
1329
|
+
}
|
|
1098
1330
|
}
|
|
1099
1331
|
|
|
1100
1332
|
private incrementDnsBucket(): void {
|
|
@@ -1165,12 +1397,16 @@ export class MetricsManager {
|
|
|
1165
1397
|
}
|
|
1166
1398
|
|
|
1167
1399
|
private pruneOldBuckets(): void {
|
|
1168
|
-
const
|
|
1400
|
+
const emailCutoff = MetricsManager.firstMinuteForHours(24);
|
|
1169
1401
|
for (const key of this.emailMinuteBuckets.keys()) {
|
|
1170
|
-
if (key <
|
|
1402
|
+
if (key < emailCutoff) {
|
|
1403
|
+
this.emailMinuteBuckets.delete(key);
|
|
1404
|
+
this.emailDirtyBuckets.delete(key);
|
|
1405
|
+
}
|
|
1171
1406
|
}
|
|
1407
|
+
const dnsCutoff = Date.now() - 86400000;
|
|
1172
1408
|
for (const key of this.dnsMinuteBuckets.keys()) {
|
|
1173
|
-
if (key <
|
|
1409
|
+
if (key < dnsCutoff) this.dnsMinuteBuckets.delete(key);
|
|
1174
1410
|
}
|
|
1175
1411
|
}
|
|
1176
1412
|
|
|
@@ -1183,20 +1419,21 @@ export class MetricsManager {
|
|
|
1183
1419
|
failed: Array<{ timestamp: number; value: number }>;
|
|
1184
1420
|
} {
|
|
1185
1421
|
this.pruneOldBuckets();
|
|
1186
|
-
const
|
|
1422
|
+
const pointCount = Math.max(
|
|
1423
|
+
1,
|
|
1424
|
+
Math.min(EMAIL_TRAFFIC_WINDOW_MINUTES, Math.floor(hours * 60)),
|
|
1425
|
+
);
|
|
1426
|
+
const end = MetricsManager.minuteKey();
|
|
1427
|
+
const start = end - (pointCount - 1) * 60_000;
|
|
1187
1428
|
const sent: Array<{ timestamp: number; value: number }> = [];
|
|
1188
1429
|
const received: Array<{ timestamp: number; value: number }> = [];
|
|
1189
1430
|
const failed: Array<{ timestamp: number; value: number }> = [];
|
|
1190
1431
|
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
.
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
const bucket = this.emailMinuteBuckets.get(key)!;
|
|
1197
|
-
sent.push({ timestamp: key, value: bucket.sent });
|
|
1198
|
-
received.push({ timestamp: key, value: bucket.received });
|
|
1199
|
-
failed.push({ timestamp: key, value: bucket.failed });
|
|
1432
|
+
for (let timestamp = start; timestamp <= end; timestamp += 60_000) {
|
|
1433
|
+
const bucket = this.emailMinuteBuckets.get(timestamp);
|
|
1434
|
+
sent.push({ timestamp, value: bucket?.sent || 0 });
|
|
1435
|
+
received.push({ timestamp, value: bucket?.received || 0 });
|
|
1436
|
+
failed.push({ timestamp, value: bucket?.failed || 0 });
|
|
1200
1437
|
}
|
|
1201
1438
|
|
|
1202
1439
|
return { sent, received, failed };
|
|
@@ -18,6 +18,11 @@ type TAdminUser = {
|
|
|
18
18
|
authSources?: Array<'local' | 'idp.global'>;
|
|
19
19
|
};
|
|
20
20
|
|
|
21
|
+
type TAuthenticatedAdmin = {
|
|
22
|
+
user: TAdminUser;
|
|
23
|
+
authSource: 'local' | 'idp.global';
|
|
24
|
+
};
|
|
25
|
+
|
|
21
26
|
export class AdminHandler {
|
|
22
27
|
public typedrouter = new plugins.typedrequest.TypedRouter();
|
|
23
28
|
|
|
@@ -330,25 +335,59 @@ export class AdminHandler {
|
|
|
330
335
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_AdminLoginWithUsernameAndPassword>(
|
|
331
336
|
'adminLoginWithUsernameAndPassword',
|
|
332
337
|
async (dataArg) => {
|
|
338
|
+
const requestedAuthSource = dataArg.authSource || 'auto';
|
|
339
|
+
let authenticatedAdmin: TAuthenticatedAdmin | null;
|
|
333
340
|
try {
|
|
334
|
-
|
|
341
|
+
authenticatedAdmin = await this.authenticateUser({
|
|
335
342
|
username: dataArg.username,
|
|
336
343
|
password: dataArg.password,
|
|
337
|
-
authSource:
|
|
344
|
+
authSource: requestedAuthSource,
|
|
338
345
|
});
|
|
339
|
-
if (!user) {
|
|
340
|
-
throw new plugins.typedrequest.TypedResponseError('login failed');
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
return {
|
|
344
|
-
identity: await this.createIdentityForUser(user),
|
|
345
|
-
};
|
|
346
346
|
} catch (error) {
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
347
|
+
await this.recordAuthenticationAttempt({
|
|
348
|
+
username: dataArg.username,
|
|
349
|
+
success: false,
|
|
350
|
+
requestedAuthSource,
|
|
351
|
+
failureReason: String((error as Error).message || '').includes('database is not ready')
|
|
352
|
+
? 'serviceUnavailable'
|
|
353
|
+
: 'internalError',
|
|
354
|
+
});
|
|
350
355
|
throw new plugins.typedrequest.TypedResponseError('login failed');
|
|
351
356
|
}
|
|
357
|
+
|
|
358
|
+
if (!authenticatedAdmin) {
|
|
359
|
+
await this.recordAuthenticationAttempt({
|
|
360
|
+
username: dataArg.username,
|
|
361
|
+
success: false,
|
|
362
|
+
requestedAuthSource,
|
|
363
|
+
failureReason: 'invalidCredentials',
|
|
364
|
+
});
|
|
365
|
+
throw new plugins.typedrequest.TypedResponseError('login failed');
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
let identity: interfaces.data.IIdentity;
|
|
369
|
+
try {
|
|
370
|
+
identity = await this.createIdentityForUser(authenticatedAdmin.user);
|
|
371
|
+
} catch {
|
|
372
|
+
await this.recordAuthenticationAttempt({
|
|
373
|
+
username: dataArg.username,
|
|
374
|
+
userId: authenticatedAdmin.user.id,
|
|
375
|
+
success: false,
|
|
376
|
+
requestedAuthSource,
|
|
377
|
+
resolvedAuthSource: authenticatedAdmin.authSource,
|
|
378
|
+
failureReason: 'identityIssuanceFailed',
|
|
379
|
+
});
|
|
380
|
+
throw new plugins.typedrequest.TypedResponseError('login failed');
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
await this.recordAuthenticationAttempt({
|
|
384
|
+
username: dataArg.username,
|
|
385
|
+
userId: authenticatedAdmin.user.id,
|
|
386
|
+
success: true,
|
|
387
|
+
requestedAuthSource,
|
|
388
|
+
resolvedAuthSource: authenticatedAdmin.authSource,
|
|
389
|
+
});
|
|
390
|
+
return { identity };
|
|
352
391
|
}
|
|
353
392
|
)
|
|
354
393
|
);
|
|
@@ -492,7 +531,7 @@ export class AdminHandler {
|
|
|
492
531
|
username: string;
|
|
493
532
|
password: string;
|
|
494
533
|
authSource?: interfaces.requests.TAdminLoginAuthSource;
|
|
495
|
-
}): Promise<
|
|
534
|
+
}): Promise<TAuthenticatedAdmin | null> {
|
|
496
535
|
const accountState = await this.getPersistentAccountState();
|
|
497
536
|
if (accountState.dbEnabled && !accountState.dbReady) {
|
|
498
537
|
throw new plugins.typedrequest.TypedResponseError('database is not ready');
|
|
@@ -508,17 +547,37 @@ export class AdminHandler {
|
|
|
508
547
|
password: optionsArg.password,
|
|
509
548
|
authSource: optionsArg.authSource || 'auto',
|
|
510
549
|
});
|
|
511
|
-
return result
|
|
550
|
+
return result
|
|
551
|
+
? { user: this.accountToUser(result.account), authSource: result.authSource }
|
|
552
|
+
: null;
|
|
512
553
|
}
|
|
513
554
|
|
|
514
555
|
for (const [_, userData] of this.users) {
|
|
515
556
|
if (userData.username === optionsArg.username && userData.password === optionsArg.password) {
|
|
516
|
-
return userData;
|
|
557
|
+
return { user: userData, authSource: 'local' };
|
|
517
558
|
}
|
|
518
559
|
}
|
|
519
560
|
return null;
|
|
520
561
|
}
|
|
521
562
|
|
|
563
|
+
private async recordAuthenticationAttempt(optionsArg: {
|
|
564
|
+
username: string;
|
|
565
|
+
userId?: string;
|
|
566
|
+
success: boolean;
|
|
567
|
+
requestedAuthSource: interfaces.requests.TAdminLoginAuthSource;
|
|
568
|
+
resolvedAuthSource?: 'local' | 'idp.global';
|
|
569
|
+
failureReason?: interfaces.data.TAuthenticationFailureReason;
|
|
570
|
+
}): Promise<void> {
|
|
571
|
+
const eventManager = this.opsServerRef.dcRouterRef.authenticationEventManager;
|
|
572
|
+
if (!eventManager) return;
|
|
573
|
+
try {
|
|
574
|
+
await eventManager.recordAttempt(optionsArg);
|
|
575
|
+
} catch (error) {
|
|
576
|
+
// Authentication availability must not depend on telemetry persistence.
|
|
577
|
+
console.warn(`Authentication event recording unavailable: ${(error as Error).message}`);
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
|
|
522
581
|
private async resolveUser(userIdArg: string): Promise<TAdminUser | null> {
|
|
523
582
|
const accountState = await this.getPersistentAccountState();
|
|
524
583
|
if (accountState.dbEnabled && !accountState.dbReady) {
|
|
@@ -27,6 +27,9 @@ export class SecurityHandler {
|
|
|
27
27
|
malwareDetected: metrics.malwareDetected,
|
|
28
28
|
phishingDetected: metrics.phishingDetected,
|
|
29
29
|
authenticationFailures: metrics.authFailures,
|
|
30
|
+
authenticationSuccesses: metrics.authSuccesses,
|
|
31
|
+
authenticationWindowStart: metrics.authWindowStart,
|
|
32
|
+
authenticationEvents: metrics.authEvents,
|
|
30
33
|
suspiciousActivities: metrics.suspiciousActivities,
|
|
31
34
|
},
|
|
32
35
|
trends: dataArg.includeDetails ? {
|
|
@@ -309,6 +312,9 @@ export class SecurityHandler {
|
|
|
309
312
|
malwareDetected: number;
|
|
310
313
|
phishingDetected: number;
|
|
311
314
|
authFailures: number;
|
|
315
|
+
authSuccesses: number;
|
|
316
|
+
authWindowStart: number;
|
|
317
|
+
authEvents: interfaces.data.IAuthenticationEvent[];
|
|
312
318
|
suspiciousActivities: number;
|
|
313
319
|
trends: {
|
|
314
320
|
spam: Array<{ timestamp: number; value: number }>;
|
|
@@ -316,6 +322,15 @@ export class SecurityHandler {
|
|
|
316
322
|
phishing: Array<{ timestamp: number; value: number }>;
|
|
317
323
|
};
|
|
318
324
|
}> {
|
|
325
|
+
const authenticationEventManager = this.opsServerRef.dcRouterRef.authenticationEventManager;
|
|
326
|
+
const authentication = authenticationEventManager
|
|
327
|
+
? await authenticationEventManager.getWindowStats()
|
|
328
|
+
: {
|
|
329
|
+
windowStart: Date.now() - 86_400_000,
|
|
330
|
+
successes: 0,
|
|
331
|
+
failures: 0,
|
|
332
|
+
events: [],
|
|
333
|
+
};
|
|
319
334
|
// Get metrics from MetricsManager if available
|
|
320
335
|
if (this.opsServerRef.dcRouterRef.metricsManager) {
|
|
321
336
|
const securityStats = await this.opsServerRef.dcRouterRef.metricsManager.getSecurityStats();
|
|
@@ -328,7 +343,10 @@ export class SecurityHandler {
|
|
|
328
343
|
},
|
|
329
344
|
malwareDetected: securityStats.malwareDetected,
|
|
330
345
|
phishingDetected: securityStats.phishingDetected,
|
|
331
|
-
authFailures:
|
|
346
|
+
authFailures: authentication.failures,
|
|
347
|
+
authSuccesses: authentication.successes,
|
|
348
|
+
authWindowStart: authentication.windowStart,
|
|
349
|
+
authEvents: authentication.events,
|
|
332
350
|
suspiciousActivities: 0,
|
|
333
351
|
trends: {
|
|
334
352
|
spam: [],
|
|
@@ -348,7 +366,10 @@ export class SecurityHandler {
|
|
|
348
366
|
},
|
|
349
367
|
malwareDetected: 0,
|
|
350
368
|
phishingDetected: 0,
|
|
351
|
-
authFailures:
|
|
369
|
+
authFailures: authentication.failures,
|
|
370
|
+
authSuccesses: authentication.successes,
|
|
371
|
+
authWindowStart: authentication.windowStart,
|
|
372
|
+
authEvents: authentication.events,
|
|
352
373
|
suspiciousActivities: 0,
|
|
353
374
|
trends: {
|
|
354
375
|
spam: [],
|