@serve.zone/dcrouter 17.0.0 → 17.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.smartconfig.json +5 -2
- package/deno.json +1 -1
- package/dist_serve/bundle.js +1316 -1151
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.dcrouter.d.ts +3 -1
- package/dist_ts/classes.dcrouter.js +110 -49
- package/dist_ts/db/classes.cache.cleaner.d.ts +3 -0
- package/dist_ts/db/classes.cache.cleaner.js +7 -3
- package/dist_ts/db/classes.dcrouter-db.js +1 -2
- package/dist_ts/db/documents/classes.cached.email.d.ts +41 -4
- package/dist_ts/db/documents/classes.cached.email.js +36 -5
- package/dist_ts/db/documents/classes.dns-record.doc.js +3 -1
- package/dist_ts/dns/classes.dns-server-runtime.d.ts +0 -1
- package/dist_ts/dns/classes.dns-server-runtime.js +4 -36
- package/dist_ts/dns/index.d.ts +1 -0
- package/dist_ts/dns/index.js +2 -1
- package/dist_ts/dns/manager.dns.d.ts +1 -0
- package/dist_ts/dns/manager.dns.js +49 -14
- package/dist_ts/dns/providers/cloudflare.provider.js +19 -4
- package/dist_ts/dns/txt-record-presentation.d.ts +20 -0
- package/dist_ts/dns/txt-record-presentation.js +131 -0
- package/dist_ts/email/classes.accepted-email-spool.d.ts +32 -3
- package/dist_ts/email/classes.accepted-email-spool.js +174 -13
- package/dist_ts/email/classes.email-domain.manager.d.ts +14 -1
- package/dist_ts/email/classes.email-domain.manager.js +151 -8
- package/dist_ts/email/classes.email-settings.manager.js +2 -2
- package/dist_ts/email/classes.mail-dns-sync.d.ts +31 -5
- package/dist_ts/email/classes.mail-dns-sync.js +508 -117
- package/dist_ts/email/classes.smartmta-blob-storage-manager.d.ts +2 -2
- package/dist_ts/email/classes.smartmta-blob-storage-manager.js +4 -3
- package/dist_ts/email/classes.workapp-mail-manager.d.ts +1 -0
- package/dist_ts/email/classes.workapp-mail-manager.js +26 -2
- package/dist_ts/email/index.d.ts +2 -0
- package/dist_ts/email/index.js +3 -1
- package/dist_ts/email/mail-dns-txt-policy.d.ts +18 -0
- package/dist_ts/email/mail-dns-txt-policy.js +466 -0
- package/dist_ts/email/reconcile-mail-dns-singleton.d.ts +47 -0
- package/dist_ts/email/reconcile-mail-dns-singleton.js +159 -0
- package/dist_ts/opsserver/classes.opsserver.d.ts +8 -0
- package/dist_ts/opsserver/classes.opsserver.js +34 -1
- package/dist_ts/opsserver/handlers/certificate.handler.js +25 -4
- package/dist_ts/opsserver/handlers/config.handler.js +9 -5
- package/dist_ts/opsserver/handlers/dns-provider.handler.js +19 -2
- package/dist_ts/opsserver/handlers/dns-record.handler.js +25 -4
- package/dist_ts/opsserver/handlers/domain.handler.js +37 -3
- package/dist_ts/opsserver/handlers/email-domain.handler.js +5 -7
- package/dist_ts/opsserver/handlers/email-ops.handler.d.ts +1 -0
- package/dist_ts/opsserver/handlers/email-ops.handler.js +26 -7
- package/dist_ts/opsserver/handlers/index.d.ts +1 -0
- package/dist_ts/opsserver/handlers/index.js +2 -1
- package/dist_ts/opsserver/handlers/logs.handler.d.ts +1 -1
- package/dist_ts/opsserver/handlers/logs.handler.js +3 -3
- package/dist_ts/opsserver/handlers/realtime.handler.d.ts +37 -0
- package/dist_ts/opsserver/handlers/realtime.handler.js +154 -0
- package/dist_ts/opsserver/handlers/remoteingress.handler.js +22 -1
- package/dist_ts/opsserver/handlers/route-management.handler.js +47 -5
- package/dist_ts/paths.d.ts +0 -11
- package/dist_ts/paths.js +1 -22
- package/dist_ts/plugins.d.ts +0 -34
- package/dist_ts/plugins.js +1 -62
- package/dist_ts_interfaces/data/email-domain.d.ts +14 -0
- package/dist_ts_interfaces/requests/config.d.ts +1 -1
- package/dist_ts_interfaces/requests/email-ops.d.ts +34 -1
- package/dist_ts_interfaces/requests/index.d.ts +1 -0
- package/dist_ts_interfaces/requests/index.js +2 -1
- package/dist_ts_interfaces/requests/realtime.d.ts +38 -0
- package/dist_ts_interfaces/requests/realtime.js +11 -0
- package/dist_ts_migrations/index.d.ts +6 -0
- package/dist_ts_migrations/index.js +204 -9
- package/dist_ts_web/00_commitinfo_data.js +1 -1
- package/dist_ts_web/appstate/certificates.js +9 -2
- package/dist_ts_web/appstate/domains.js +16 -5
- package/dist_ts_web/appstate/email-domains.d.ts +3 -0
- package/dist_ts_web/appstate/email-domains.js +47 -14
- package/dist_ts_web/appstate/email-ops.d.ts +1 -0
- package/dist_ts_web/appstate/email-ops.js +20 -2
- package/dist_ts_web/appstate/realtime.d.ts +27 -0
- package/dist_ts_web/appstate/realtime.js +71 -0
- package/dist_ts_web/appstate/remoteingress.js +9 -3
- package/dist_ts_web/appstate/routes.js +16 -5
- package/dist_ts_web/appstate/runtime.js +135 -7
- package/dist_ts_web/appstate.d.ts +1 -0
- package/dist_ts_web/appstate.js +2 -1
- package/dist_ts_web/elements/email/email-domain-creation-progress.d.ts +33 -0
- package/dist_ts_web/elements/email/email-domain-creation-progress.js +257 -0
- package/dist_ts_web/elements/email/ops-view-email-domains.js +233 -68
- package/dist_ts_web/elements/email/ops-view-emails.d.ts +4 -2
- package/dist_ts_web/elements/email/ops-view-emails.js +25 -13
- package/dist_ts_web/elements/overview/ops-view-config.js +5 -5
- package/package.json +22 -19
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.dcrouter.ts +121 -54
- package/ts/db/classes.cache.cleaner.ts +18 -3
- package/ts/db/classes.dcrouter-db.ts +0 -2
- package/ts/db/documents/classes.cached.email.ts +78 -4
- package/ts/db/documents/classes.dns-record.doc.ts +1 -0
- package/ts/dns/classes.dns-server-runtime.ts +9 -45
- package/ts/dns/index.ts +1 -0
- package/ts/dns/manager.dns.ts +50 -16
- package/ts/dns/providers/cloudflare.provider.ts +21 -3
- package/ts/dns/txt-record-presentation.ts +134 -0
- package/ts/email/classes.accepted-email-spool.ts +220 -14
- package/ts/email/classes.email-domain.manager.ts +201 -7
- package/ts/email/classes.email-settings.manager.ts +1 -1
- package/ts/email/classes.mail-dns-sync.ts +588 -122
- package/ts/email/classes.smartmta-blob-storage-manager.ts +3 -2
- package/ts/email/classes.workapp-mail-manager.ts +34 -1
- package/ts/email/index.ts +2 -0
- package/ts/email/mail-dns-txt-policy.ts +505 -0
- package/ts/email/reconcile-mail-dns-singleton.ts +224 -0
- package/ts/opsserver/classes.opsserver.ts +42 -0
- package/ts/opsserver/handlers/certificate.handler.ts +24 -3
- package/ts/opsserver/handlers/config.handler.ts +9 -5
- package/ts/opsserver/handlers/dns-provider.handler.ts +18 -1
- package/ts/opsserver/handlers/dns-record.handler.ts +24 -3
- package/ts/opsserver/handlers/domain.handler.ts +36 -2
- package/ts/opsserver/handlers/email-domain.handler.ts +4 -5
- package/ts/opsserver/handlers/email-ops.handler.ts +29 -6
- package/ts/opsserver/handlers/index.ts +1 -0
- package/ts/opsserver/handlers/logs.handler.ts +2 -2
- package/ts/opsserver/handlers/realtime.handler.ts +222 -0
- package/ts/opsserver/handlers/remoteingress.handler.ts +21 -0
- package/ts/opsserver/handlers/route-management.handler.ts +46 -4
- package/ts/paths.ts +0 -25
- package/ts/plugins.ts +0 -68
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/appstate/certificates.ts +6 -1
- package/ts_web/appstate/domains.ts +11 -4
- package/ts_web/appstate/email-domains.ts +68 -14
- package/ts_web/appstate/email-ops.ts +20 -1
- package/ts_web/appstate/realtime.ts +109 -0
- package/ts_web/appstate/remoteingress.ts +6 -2
- package/ts_web/appstate/routes.ts +11 -4
- package/ts_web/appstate/runtime.ts +176 -7
- package/ts_web/appstate.ts +1 -0
- package/ts_web/elements/email/email-domain-creation-progress.ts +340 -0
- package/ts_web/elements/email/ops-view-email-domains.ts +273 -74
- package/ts_web/elements/email/ops-view-emails.ts +23 -16
- package/ts_web/elements/overview/ops-view-config.ts +4 -4
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@serve.zone/dcrouter",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "17.
|
|
4
|
+
"version": "17.1.0",
|
|
5
5
|
"description": "A multifaceted routing service handling mail and SMS delivery functions.",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"bin": {
|
|
@@ -27,48 +27,48 @@
|
|
|
27
27
|
"dependencies": {
|
|
28
28
|
"@api.global/typedrequest": "^3.3.2",
|
|
29
29
|
"@api.global/typedrequest-interfaces": "^3.0.19",
|
|
30
|
-
"@api.global/typedserver": "^8.
|
|
31
|
-
"@api.global/typedsocket": "^
|
|
30
|
+
"@api.global/typedserver": "^8.6.4",
|
|
31
|
+
"@api.global/typedsocket": "^5.0.0",
|
|
32
32
|
"@apiclient.xyz/cloudflare": "^7.1.0",
|
|
33
33
|
"@design.estate/dees-catalog": "^3.96.3",
|
|
34
|
-
"@design.estate/dees-element": "^2.
|
|
35
|
-
"@idp.global/sdk": "^
|
|
34
|
+
"@design.estate/dees-element": "^2.4.0",
|
|
35
|
+
"@idp.global/sdk": "^14.3.0",
|
|
36
36
|
"@push.rocks/lik": "^6.4.1",
|
|
37
37
|
"@push.rocks/projectinfo": "^5.1.0",
|
|
38
38
|
"@push.rocks/qenv": "^6.1.4",
|
|
39
39
|
"@push.rocks/smartacme": "^9.5.0",
|
|
40
40
|
"@push.rocks/smartbucket": "^4.6.1",
|
|
41
41
|
"@push.rocks/smartchallenge": "^1.1.0",
|
|
42
|
-
"@push.rocks/smartdata": "^7.
|
|
42
|
+
"@push.rocks/smartdata": "^7.3.1",
|
|
43
43
|
"@push.rocks/smartdb": "^2.12.3",
|
|
44
|
-
"@push.rocks/smartdns": "^7.
|
|
45
|
-
"@push.rocks/smartfs": "^1.
|
|
46
|
-
"@push.rocks/smartguard": "^3.1.
|
|
47
|
-
"@push.rocks/smartjwt": "^2.
|
|
44
|
+
"@push.rocks/smartdns": "^7.12.0",
|
|
45
|
+
"@push.rocks/smartfs": "^1.6.0",
|
|
46
|
+
"@push.rocks/smartguard": "^3.1.2",
|
|
47
|
+
"@push.rocks/smartjwt": "^2.5.1",
|
|
48
48
|
"@push.rocks/smartlog": "^3.2.2",
|
|
49
49
|
"@push.rocks/smartmcp": "^0.2.0",
|
|
50
50
|
"@push.rocks/smartmetrics": "^3.0.3",
|
|
51
51
|
"@push.rocks/smartmigration": "1.4.1",
|
|
52
|
-
"@push.rocks/smartmta": "^
|
|
52
|
+
"@push.rocks/smartmta": "^8.0.1",
|
|
53
53
|
"@push.rocks/smartnetwork": "^4.8.0",
|
|
54
54
|
"@push.rocks/smartpath": "^6.0.0",
|
|
55
55
|
"@push.rocks/smartpromise": "^4.2.4",
|
|
56
56
|
"@push.rocks/smartproxy": "^27.17.5",
|
|
57
57
|
"@push.rocks/smartradius": "^1.3.0",
|
|
58
|
-
"@push.rocks/smartrequest": "^5.0.
|
|
59
|
-
"@push.rocks/smartrx": "^3.0.
|
|
58
|
+
"@push.rocks/smartrequest": "^5.0.4",
|
|
59
|
+
"@push.rocks/smartrx": "^3.0.12",
|
|
60
60
|
"@push.rocks/smartstate": "^2.3.1",
|
|
61
|
-
"@push.rocks/smartunique": "^3.0.
|
|
62
|
-
"@push.rocks/smartvpn": "1.
|
|
61
|
+
"@push.rocks/smartunique": "^3.0.10",
|
|
62
|
+
"@push.rocks/smartvpn": "1.21.0",
|
|
63
63
|
"@push.rocks/taskbuffer": "^8.0.2",
|
|
64
|
-
"@serve.zone/catalog": "^2.
|
|
65
|
-
"@serve.zone/interfaces": "^
|
|
64
|
+
"@serve.zone/catalog": "^2.15.0",
|
|
65
|
+
"@serve.zone/interfaces": "^16.3.0",
|
|
66
66
|
"@serve.zone/remoteingress": "^5.0.0",
|
|
67
67
|
"@tsclass/tsclass": "^9.5.1",
|
|
68
68
|
"@types/qrcode": "^1.5.6",
|
|
69
|
-
"lru-cache": "^11.5.
|
|
69
|
+
"lru-cache": "^11.5.2",
|
|
70
70
|
"qrcode": "^1.5.4",
|
|
71
|
-
"uuid": "^14.0.
|
|
71
|
+
"uuid": "^14.0.1"
|
|
72
72
|
},
|
|
73
73
|
"keywords": [
|
|
74
74
|
"mail service",
|
|
@@ -114,6 +114,9 @@
|
|
|
114
114
|
".smartconfig.json",
|
|
115
115
|
"readme.md"
|
|
116
116
|
],
|
|
117
|
+
"engines": {
|
|
118
|
+
"node": ">=22"
|
|
119
|
+
},
|
|
117
120
|
"scripts": {
|
|
118
121
|
"test": "(tstest test/ --verbose --logfile --timeout 60)",
|
|
119
122
|
"start": "(node ./cli.js)",
|
package/ts/00_commitinfo_data.ts
CHANGED
package/ts/classes.dcrouter.ts
CHANGED
|
@@ -283,6 +283,11 @@ export class DcRouter {
|
|
|
283
283
|
|
|
284
284
|
public storageManager!: IStorageManagerLike;
|
|
285
285
|
private smartMtaBlobStorageManager?: SmartMtaBlobStorageManager;
|
|
286
|
+
private databaseMigrationsReady = false;
|
|
287
|
+
|
|
288
|
+
public getSmartMtaBlobStorageManager(): SmartMtaBlobStorageManager | undefined {
|
|
289
|
+
return this.smartMtaBlobStorageManager;
|
|
290
|
+
}
|
|
286
291
|
|
|
287
292
|
// Unified database (smartdata + LocalSmartDb or external MongoDB)
|
|
288
293
|
public dcRouterDb?: DcRouterDb;
|
|
@@ -376,7 +381,6 @@ export class DcRouter {
|
|
|
376
381
|
|
|
377
382
|
// Resolve all data paths from baseDir
|
|
378
383
|
this.resolvedPaths = paths.resolvePaths(this.options.baseDir);
|
|
379
|
-
paths.ensureDataDirectories(this.resolvedPaths);
|
|
380
384
|
this.workAppMailManager = new WorkAppMailManager(this);
|
|
381
385
|
this.acceptedEmailSpool = new AcceptedEmailSpool(this);
|
|
382
386
|
const mailEgressIdentitySource = new ReleasedRemoteIngressEgressIdentitySource(this);
|
|
@@ -427,6 +431,7 @@ export class DcRouter {
|
|
|
427
431
|
await this.setupDcRouterDb();
|
|
428
432
|
})
|
|
429
433
|
.withStop(async () => {
|
|
434
|
+
this.databaseMigrationsReady = false;
|
|
430
435
|
if (this.cacheCleaner) {
|
|
431
436
|
this.cacheCleaner.stop();
|
|
432
437
|
this.cacheCleaner = undefined;
|
|
@@ -1013,6 +1018,7 @@ export class DcRouter {
|
|
|
1013
1018
|
*/
|
|
1014
1019
|
private async setupDcRouterDb(): Promise<void> {
|
|
1015
1020
|
logger.log('info', 'Setting up DcRouterDb...');
|
|
1021
|
+
this.databaseMigrationsReady = false;
|
|
1016
1022
|
|
|
1017
1023
|
const dbConfig = this.options.dbConfig || {};
|
|
1018
1024
|
|
|
@@ -1024,39 +1030,70 @@ export class DcRouter {
|
|
|
1024
1030
|
debug: false,
|
|
1025
1031
|
});
|
|
1026
1032
|
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
legacySmartMtaStoragePath: plugins.path.join(
|
|
1035
|
-
this.resolvedPaths.dataDir,
|
|
1036
|
-
'smartmta-storage',
|
|
1037
|
-
),
|
|
1038
|
-
});
|
|
1039
|
-
const migrationResult = await migration.run();
|
|
1040
|
-
if (migrationResult.stepsApplied.length > 0) {
|
|
1041
|
-
logger.log('info',
|
|
1042
|
-
`smartmigration: ${migrationResult.currentVersionBefore ?? 'fresh'} → ${migrationResult.currentVersionAfter} ` +
|
|
1043
|
-
`(${migrationResult.stepsApplied.length} step(s) applied in ${migrationResult.totalDurationMs}ms)`,
|
|
1044
|
-
);
|
|
1045
|
-
} else if (migrationResult.wasFreshInstall) {
|
|
1046
|
-
logger.log('info', `smartmigration: fresh install stamped to ${migrationResult.currentVersionAfter}`);
|
|
1047
|
-
}
|
|
1033
|
+
try {
|
|
1034
|
+
await this.dcRouterDb.start();
|
|
1035
|
+
if (this.options.emailBlobStorage && !this.smartMtaBlobStorageManager) {
|
|
1036
|
+
this.smartMtaBlobStorageManager = await SmartMtaBlobStorageManager.create(
|
|
1037
|
+
this.options.emailBlobStorage,
|
|
1038
|
+
);
|
|
1039
|
+
}
|
|
1048
1040
|
|
|
1049
|
-
|
|
1041
|
+
// Run any pending data migrations before anything else reads from the DB.
|
|
1042
|
+
// This must complete before ConfigManagers loads profiles.
|
|
1043
|
+
const migration = await createMigrationRunner(this.dcRouterDb.getDb(), commitinfo.version, {
|
|
1044
|
+
remoteIngressHubSettings: this.getRemoteIngressHubSettingsMigrationSeed(),
|
|
1045
|
+
emailServerSettings: this.getEmailSettingsMigrationSeed(),
|
|
1046
|
+
legacySmartMtaStoragePath: plugins.path.join(
|
|
1047
|
+
this.resolvedPaths.dataDir,
|
|
1048
|
+
'smartmta-storage',
|
|
1049
|
+
),
|
|
1050
|
+
legacyRawMessageBlobStorage: this.smartMtaBlobStorageManager,
|
|
1051
|
+
});
|
|
1052
|
+
const migrationResult = await migration.run();
|
|
1053
|
+
if (migrationResult.stepsApplied.length > 0) {
|
|
1054
|
+
logger.log('info',
|
|
1055
|
+
`smartmigration: ${migrationResult.currentVersionBefore ?? 'fresh'} → ${migrationResult.currentVersionAfter} ` +
|
|
1056
|
+
`(${migrationResult.stepsApplied.length} step(s) applied in ${migrationResult.totalDurationMs}ms)`,
|
|
1057
|
+
);
|
|
1058
|
+
} else if (migrationResult.wasFreshInstall) {
|
|
1059
|
+
logger.log('info', `smartmigration: fresh install stamped to ${migrationResult.currentVersionAfter}`);
|
|
1060
|
+
}
|
|
1050
1061
|
|
|
1051
|
-
|
|
1052
|
-
const cleanupIntervalMs = (dbConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
|
|
1053
|
-
this.cacheCleaner = new CacheCleaner(this.dcRouterDb, {
|
|
1054
|
-
intervalMs: cleanupIntervalMs,
|
|
1055
|
-
verbose: false,
|
|
1056
|
-
});
|
|
1057
|
-
this.cacheCleaner.start();
|
|
1062
|
+
this.storageManager = new SmartMtaStorageManager(this.dcRouterDb.getDb());
|
|
1058
1063
|
|
|
1059
|
-
|
|
1064
|
+
// Start the cache cleaner for TTL-based document cleanup
|
|
1065
|
+
const cleanupIntervalMs = (dbConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
|
|
1066
|
+
this.cacheCleaner = new CacheCleaner(this.dcRouterDb, {
|
|
1067
|
+
intervalMs: cleanupIntervalMs,
|
|
1068
|
+
verbose: false,
|
|
1069
|
+
beforeDeleteCachedEmail: async (cachedEmail) => {
|
|
1070
|
+
await this.acceptedEmailSpool.deleteRawMessage(cachedEmail);
|
|
1071
|
+
},
|
|
1072
|
+
});
|
|
1073
|
+
this.cacheCleaner.start();
|
|
1074
|
+
|
|
1075
|
+
this.databaseMigrationsReady = true;
|
|
1076
|
+
logger.log('info', `DcRouterDb ready (${this.dcRouterDb.isEmbedded() ? 'embedded' : 'external'})`);
|
|
1077
|
+
} catch (error) {
|
|
1078
|
+
this.databaseMigrationsReady = false;
|
|
1079
|
+
if (this.cacheCleaner) {
|
|
1080
|
+
this.cacheCleaner.stop();
|
|
1081
|
+
this.cacheCleaner = undefined;
|
|
1082
|
+
}
|
|
1083
|
+
const failedDb = this.dcRouterDb;
|
|
1084
|
+
this.dcRouterDb = undefined;
|
|
1085
|
+
try {
|
|
1086
|
+
await failedDb?.stop();
|
|
1087
|
+
} catch (stopError: unknown) {
|
|
1088
|
+
logger.log(
|
|
1089
|
+
'warn',
|
|
1090
|
+
`Failed to stop DcRouterDb after startup migration failure: ${(stopError as Error).message}`,
|
|
1091
|
+
);
|
|
1092
|
+
} finally {
|
|
1093
|
+
DcRouterDb.resetInstance();
|
|
1094
|
+
}
|
|
1095
|
+
throw error;
|
|
1096
|
+
}
|
|
1060
1097
|
}
|
|
1061
1098
|
|
|
1062
1099
|
/**
|
|
@@ -1639,6 +1676,9 @@ export class DcRouter {
|
|
|
1639
1676
|
}
|
|
1640
1677
|
|
|
1641
1678
|
// Apply port mapping if behind SmartProxy
|
|
1679
|
+
if (!this.databaseMigrationsReady) {
|
|
1680
|
+
throw new Error('DcRouterDb migrations must complete before email acceptance');
|
|
1681
|
+
}
|
|
1642
1682
|
const portMapping = this.options.emailPortConfig?.portMapping || {
|
|
1643
1683
|
25: 10025, // SMTP
|
|
1644
1684
|
587: 10587, // Submission
|
|
@@ -1660,25 +1700,21 @@ export class DcRouter {
|
|
|
1660
1700
|
delete queueBehavior.storageManager;
|
|
1661
1701
|
delete queueBehavior.storageType;
|
|
1662
1702
|
delete queueBehavior.persistentPath;
|
|
1663
|
-
if (
|
|
1664
|
-
|
|
1665
|
-
|
|
1703
|
+
if (!this.smartMtaBlobStorageManager) {
|
|
1704
|
+
throw new Error(
|
|
1705
|
+
'emailBlobStorage is required for durable SmartMTA queue and raw RFC822 persistence',
|
|
1666
1706
|
);
|
|
1667
1707
|
}
|
|
1668
|
-
const queueOptions: IUnifiedEmailServerOptions['queue'] =
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
}
|
|
1674
|
-
: {
|
|
1675
|
-
...queueBehavior,
|
|
1676
|
-
storageMode: 'memory',
|
|
1677
|
-
};
|
|
1708
|
+
const queueOptions: IUnifiedEmailServerOptions['queue'] = {
|
|
1709
|
+
...queueBehavior,
|
|
1710
|
+
storageMode: 'managed',
|
|
1711
|
+
storageManager: this.smartMtaBlobStorageManager,
|
|
1712
|
+
};
|
|
1678
1713
|
|
|
1679
|
-
|
|
1714
|
+
let emailConfig: IUnifiedEmailServerOptions = await this.workAppMailManager.applyStoredIdentitiesToEmailConfig({
|
|
1680
1715
|
...this.options.emailConfig,
|
|
1681
1716
|
ports: mappedEmailPorts,
|
|
1717
|
+
dkimKeyProvisioning: 'caller-managed',
|
|
1682
1718
|
persistRoutes: this.options.emailConfig.persistRoutes ?? false,
|
|
1683
1719
|
queue: queueOptions,
|
|
1684
1720
|
outbound: {
|
|
@@ -1724,9 +1760,28 @@ export class DcRouter {
|
|
|
1724
1760
|
});
|
|
1725
1761
|
|
|
1726
1762
|
// Create unified email server
|
|
1727
|
-
|
|
1763
|
+
let emailServer = new UnifiedEmailServer(this, emailConfig);
|
|
1728
1764
|
this.emailServer = emailServer;
|
|
1729
1765
|
this.clearEmailEventSubscriptions();
|
|
1766
|
+
try {
|
|
1767
|
+
const repairResult = await this.emailDomainManager?.repairManagedDkimBeforeEmailStart();
|
|
1768
|
+
if (repairResult && repairResult.failedDomainIds.length > 0) {
|
|
1769
|
+
emailConfig = await this.workAppMailManager.applyStoredIdentitiesToEmailConfig({
|
|
1770
|
+
...emailConfig,
|
|
1771
|
+
domains: [...(this.options.emailConfig?.domains || [])],
|
|
1772
|
+
});
|
|
1773
|
+
// DkimManager captures its DomainRegistry at construction. Rebuild the
|
|
1774
|
+
// unstarted server so per-domain repair failures are absent from
|
|
1775
|
+
// caller-managed DKIM startup validation.
|
|
1776
|
+
emailServer = new UnifiedEmailServer(this, emailConfig);
|
|
1777
|
+
this.emailServer = emailServer;
|
|
1778
|
+
}
|
|
1779
|
+
} catch (error) {
|
|
1780
|
+
if (this.emailServer === emailServer) {
|
|
1781
|
+
this.emailServer = undefined;
|
|
1782
|
+
}
|
|
1783
|
+
throw error;
|
|
1784
|
+
}
|
|
1730
1785
|
|
|
1731
1786
|
// Set up error handling
|
|
1732
1787
|
this.addEmailEventSubscription(emailServer, 'error', (err: Error) => {
|
|
@@ -1749,6 +1804,10 @@ export class DcRouter {
|
|
|
1749
1804
|
// Start the server
|
|
1750
1805
|
try {
|
|
1751
1806
|
await emailServer.start();
|
|
1807
|
+
// Restore failed managed domains as inbound-only after DKIM validation.
|
|
1808
|
+
// Outbound routes and SMTP users stay absent through readiness gating.
|
|
1809
|
+
await this.emailDomainManager?.syncManagedDomainsToRuntime();
|
|
1810
|
+
await this.workAppMailManager.applyStoredIdentitiesToRuntime();
|
|
1752
1811
|
} catch (error: unknown) {
|
|
1753
1812
|
this.clearEmailEventSubscriptions();
|
|
1754
1813
|
try {
|
|
@@ -1763,18 +1822,25 @@ export class DcRouter {
|
|
|
1763
1822
|
}
|
|
1764
1823
|
|
|
1765
1824
|
try {
|
|
1766
|
-
this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemEnqueued', (item: TSmartMtaQueueItemLike) => {
|
|
1767
|
-
this.acceptedEmailSpool.trackQueueUpdate(item, 'queued', 'Unable to update accepted email after queue enqueue');
|
|
1825
|
+
this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemEnqueued', async (item: TSmartMtaQueueItemLike) => {
|
|
1826
|
+
await this.acceptedEmailSpool.trackQueueUpdate(item, 'queued', 'Unable to update accepted email after queue enqueue');
|
|
1768
1827
|
});
|
|
1769
|
-
this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemDelivered', (item: TSmartMtaQueueItemLike) => {
|
|
1770
|
-
this.acceptedEmailSpool.trackQueueUpdate(item, 'delivered', 'Unable to mark accepted email delivered');
|
|
1828
|
+
this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemDelivered', async (item: TSmartMtaQueueItemLike) => {
|
|
1829
|
+
await this.acceptedEmailSpool.trackQueueUpdate(item, 'delivered', 'Unable to mark accepted email delivered');
|
|
1771
1830
|
});
|
|
1772
|
-
this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemDeferred', (item: TSmartMtaQueueItemLike) => {
|
|
1773
|
-
this.acceptedEmailSpool.trackQueueUpdate(item, 'deferred', 'Unable to defer accepted email');
|
|
1831
|
+
this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemDeferred', async (item: TSmartMtaQueueItemLike) => {
|
|
1832
|
+
await this.acceptedEmailSpool.trackQueueUpdate(item, 'deferred', 'Unable to defer accepted email');
|
|
1774
1833
|
});
|
|
1775
|
-
this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemFailed', (item: TSmartMtaQueueItemLike) => {
|
|
1776
|
-
this.acceptedEmailSpool.trackQueueUpdate(item, 'failed', 'Unable to mark accepted email failed');
|
|
1834
|
+
this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemFailed', async (item: TSmartMtaQueueItemLike) => {
|
|
1835
|
+
await this.acceptedEmailSpool.trackQueueUpdate(item, 'failed', 'Unable to mark accepted email failed');
|
|
1777
1836
|
});
|
|
1837
|
+
this.addEmailEventSubscription(
|
|
1838
|
+
emailServer.deliverySystem,
|
|
1839
|
+
'smtpTransactionCompleted',
|
|
1840
|
+
async (transaction: import('./db/documents/classes.cached.email.js').ICachedEmailSmtpTransaction) => {
|
|
1841
|
+
await this.acceptedEmailSpool.trackSmtpTransaction(transaction, emailServer);
|
|
1842
|
+
},
|
|
1843
|
+
);
|
|
1778
1844
|
|
|
1779
1845
|
// Wire delivery events to MetricsManager and logger using smartmta's public queue APIs.
|
|
1780
1846
|
if (this.metricsManager) {
|
|
@@ -1826,6 +1892,7 @@ export class DcRouter {
|
|
|
1826
1892
|
updateQueueSize();
|
|
1827
1893
|
}
|
|
1828
1894
|
|
|
1895
|
+
await this.acceptedEmailSpool.recoverSmtpTransactionHistory(emailServer);
|
|
1829
1896
|
await this.acceptedEmailSpool.recoverQueuedEmails();
|
|
1830
1897
|
this.acceptedEmailSpool.start();
|
|
1831
1898
|
} catch (error: unknown) {
|
|
@@ -14,6 +14,8 @@ export interface ICacheCleanerOptions {
|
|
|
14
14
|
intervalMs?: number;
|
|
15
15
|
/** Enable verbose logging */
|
|
16
16
|
verbose?: boolean;
|
|
17
|
+
/** Delete externally stored raw payloads before removing a CachedEmail row. */
|
|
18
|
+
beforeDeleteCachedEmail?: (cachedEmailArg: CachedEmail) => Promise<void>;
|
|
17
19
|
}
|
|
18
20
|
|
|
19
21
|
/**
|
|
@@ -25,7 +27,11 @@ export interface ICacheCleanerOptions {
|
|
|
25
27
|
export class CacheCleaner {
|
|
26
28
|
private cleanupInterval: ReturnType<typeof setInterval> | null = null;
|
|
27
29
|
private isRunning: boolean = false;
|
|
28
|
-
private options:
|
|
30
|
+
private options: {
|
|
31
|
+
intervalMs: number;
|
|
32
|
+
verbose: boolean;
|
|
33
|
+
beforeDeleteCachedEmail?: (cachedEmailArg: CachedEmail) => Promise<void>;
|
|
34
|
+
};
|
|
29
35
|
private dcRouterDb: DcRouterDb;
|
|
30
36
|
|
|
31
37
|
constructor(dcRouterDb: DcRouterDb, options: ICacheCleanerOptions = {}) {
|
|
@@ -33,6 +39,7 @@ export class CacheCleaner {
|
|
|
33
39
|
this.options = {
|
|
34
40
|
intervalMs: options.intervalMs || 60 * 60 * 1000, // 1 hour default
|
|
35
41
|
verbose: options.verbose || false,
|
|
42
|
+
beforeDeleteCachedEmail: options.beforeDeleteCachedEmail,
|
|
36
43
|
};
|
|
37
44
|
}
|
|
38
45
|
|
|
@@ -95,7 +102,11 @@ export class CacheCleaner {
|
|
|
95
102
|
const results: { collection: string; deleted: number }[] = [];
|
|
96
103
|
|
|
97
104
|
try {
|
|
98
|
-
const emailsDeleted = await this.cleanExpiredDocuments(
|
|
105
|
+
const emailsDeleted = await this.cleanExpiredDocuments(
|
|
106
|
+
CachedEmail,
|
|
107
|
+
now,
|
|
108
|
+
this.options.beforeDeleteCachedEmail,
|
|
109
|
+
);
|
|
99
110
|
results.push({ collection: 'CachedEmail', deleted: emailsDeleted });
|
|
100
111
|
|
|
101
112
|
const ipReputationDeleted = await this.cleanExpiredDocuments(CachedIPReputation, now);
|
|
@@ -124,7 +135,8 @@ export class CacheCleaner {
|
|
|
124
135
|
*/
|
|
125
136
|
private async cleanExpiredDocuments<T extends { delete: () => Promise<void> }>(
|
|
126
137
|
documentClass: { getInstances: (filter: any) => Promise<T[]> },
|
|
127
|
-
now: Date
|
|
138
|
+
now: Date,
|
|
139
|
+
beforeDeleteArg?: (documentArg: T) => Promise<void>,
|
|
128
140
|
): Promise<number> {
|
|
129
141
|
try {
|
|
130
142
|
// Find all expired documents
|
|
@@ -136,6 +148,9 @@ export class CacheCleaner {
|
|
|
136
148
|
let deletedCount = 0;
|
|
137
149
|
for (const doc of expiredDocs) {
|
|
138
150
|
try {
|
|
151
|
+
if (beforeDeleteArg) {
|
|
152
|
+
await beforeDeleteArg(doc);
|
|
153
|
+
}
|
|
139
154
|
await doc.delete();
|
|
140
155
|
deletedCount++;
|
|
141
156
|
} catch (deleteError: unknown) {
|
|
@@ -80,8 +80,6 @@ export class DcRouterDb {
|
|
|
80
80
|
logger.log('info', `DcRouterDb connecting to external MongoDB`);
|
|
81
81
|
} else {
|
|
82
82
|
// Embedded LocalSmartDb mode
|
|
83
|
-
await plugins.fsUtils.ensureDir(this.options.storagePath);
|
|
84
|
-
|
|
85
83
|
this.localSmartDb = new plugins.smartdb.LocalSmartDb({
|
|
86
84
|
folderPath: this.options.storagePath,
|
|
87
85
|
});
|
|
@@ -14,6 +14,54 @@ export type TCachedEmailStatus = 'pending' | 'processing' | 'queued' | 'delivere
|
|
|
14
14
|
*/
|
|
15
15
|
export type TCachedEmailDirection = 'inbound' | 'outbound';
|
|
16
16
|
|
|
17
|
+
export type TCachedEmailSmtpDeliveryPhase =
|
|
18
|
+
| 'connect'
|
|
19
|
+
| 'greeting'
|
|
20
|
+
| 'ehlo'
|
|
21
|
+
| 'starttls'
|
|
22
|
+
| 'tls_handshake'
|
|
23
|
+
| 'post_tls_ehlo'
|
|
24
|
+
| 'auth'
|
|
25
|
+
| 'mail_from'
|
|
26
|
+
| 'rcpt_to'
|
|
27
|
+
| 'data_command'
|
|
28
|
+
| 'message_body'
|
|
29
|
+
| 'final_response'
|
|
30
|
+
| 'rset'
|
|
31
|
+
| 'quit'
|
|
32
|
+
| 'unknown';
|
|
33
|
+
|
|
34
|
+
export interface ICachedEmailSmtpTransaction {
|
|
35
|
+
id: string;
|
|
36
|
+
queueItemId: string;
|
|
37
|
+
queueAttempt: number;
|
|
38
|
+
targetHost: string;
|
|
39
|
+
targetPort: number;
|
|
40
|
+
recipientDomain?: string;
|
|
41
|
+
recipients: string[];
|
|
42
|
+
startedAt: string;
|
|
43
|
+
completedAt: string;
|
|
44
|
+
durationMs: number;
|
|
45
|
+
outcome: 'succeeded' | 'failed';
|
|
46
|
+
retryable: boolean;
|
|
47
|
+
errorType?: string;
|
|
48
|
+
smtpCode?: number;
|
|
49
|
+
recipientResults?: Array<{
|
|
50
|
+
recipient: string;
|
|
51
|
+
accepted: boolean;
|
|
52
|
+
responseCode: number;
|
|
53
|
+
}>;
|
|
54
|
+
failurePhase?: TCachedEmailSmtpDeliveryPhase;
|
|
55
|
+
error?: string;
|
|
56
|
+
transcript: Array<{
|
|
57
|
+
timestampMs: number;
|
|
58
|
+
phase: TCachedEmailSmtpDeliveryPhase;
|
|
59
|
+
direction: 'client' | 'server' | 'system';
|
|
60
|
+
text: string;
|
|
61
|
+
responseCode?: number;
|
|
62
|
+
}>;
|
|
63
|
+
}
|
|
64
|
+
|
|
17
65
|
/**
|
|
18
66
|
* Helper to get the smartdata database instance
|
|
19
67
|
*/
|
|
@@ -70,11 +118,21 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
|
|
|
70
118
|
@plugins.smartdata.svDb()
|
|
71
119
|
public subject!: string;
|
|
72
120
|
|
|
73
|
-
/**
|
|
74
|
-
|
|
75
|
-
|
|
121
|
+
/** Legacy inline RFC822 content. New messages use rawContentObjectKey. */
|
|
122
|
+
@plugins.smartdata.svDb()
|
|
123
|
+
public rawContent?: string;
|
|
124
|
+
|
|
125
|
+
/** SmartBucket object key containing the raw RFC822 message. */
|
|
126
|
+
@plugins.smartdata.svDb()
|
|
127
|
+
public rawContentObjectKey?: string;
|
|
128
|
+
|
|
129
|
+
/** Raw RFC822 message size in bytes. */
|
|
76
130
|
@plugins.smartdata.svDb()
|
|
77
|
-
public
|
|
131
|
+
public rawContentSize: number = 0;
|
|
132
|
+
|
|
133
|
+
/** Bounded durable SMTP transaction history copied from SmartMTA. */
|
|
134
|
+
@plugins.smartdata.svDb()
|
|
135
|
+
public smtpTransactions: ICachedEmailSmtpTransaction[] = [];
|
|
78
136
|
|
|
79
137
|
/**
|
|
80
138
|
* Current status of the email
|
|
@@ -164,6 +222,14 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
|
|
|
164
222
|
this.bcc = [];
|
|
165
223
|
}
|
|
166
224
|
|
|
225
|
+
public override async createSavableObject(): Promise<CachedEmail> {
|
|
226
|
+
const savableObject = await super.createSavableObject();
|
|
227
|
+
if (this.rawContentObjectKey) {
|
|
228
|
+
delete savableObject.rawContent;
|
|
229
|
+
}
|
|
230
|
+
return savableObject;
|
|
231
|
+
}
|
|
232
|
+
|
|
167
233
|
/**
|
|
168
234
|
* Create a new CachedEmail with a unique ID
|
|
169
235
|
*/
|
|
@@ -286,6 +352,14 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
|
|
|
286
352
|
this.lastError = error;
|
|
287
353
|
}
|
|
288
354
|
|
|
355
|
+
public appendSmtpTransaction(transactionArg: ICachedEmailSmtpTransaction): void {
|
|
356
|
+
const transactions = (this.smtpTransactions || []).filter(
|
|
357
|
+
(transaction) => transaction.id !== transactionArg.id,
|
|
358
|
+
);
|
|
359
|
+
transactions.push(structuredClone(transactionArg));
|
|
360
|
+
this.smtpTransactions = transactions.slice(-20);
|
|
361
|
+
}
|
|
362
|
+
|
|
289
363
|
/**
|
|
290
364
|
* Increment attempt counter and schedule next attempt
|
|
291
365
|
*/
|
|
@@ -61,6 +61,7 @@ export class DnsRecordDoc extends plugins.smartdata.SmartDataDbDoc<DnsRecordDoc,
|
|
|
61
61
|
if (!this.managedBy) delete (savable as any).managedBy;
|
|
62
62
|
if (!this.managedOwnerId) delete (savable as any).managedOwnerId;
|
|
63
63
|
if (!this.managedRecordKey) delete (savable as any).managedRecordKey;
|
|
64
|
+
if (!this.providerRecordId?.trim()) delete (savable as any).providerRecordId;
|
|
64
65
|
return savable;
|
|
65
66
|
}
|
|
66
67
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import * as plugins from '../plugins.js';
|
|
2
|
-
import * as paths from '../paths.js';
|
|
3
2
|
import { logger } from '../logger.js';
|
|
4
3
|
import { buildEmailDnsRecords } from '../email/index.js';
|
|
5
4
|
import type { DcRouter } from '../classes.dcrouter.js';
|
|
@@ -130,10 +129,7 @@ export class DnsServerRuntime {
|
|
|
130
129
|
// Generate email DNS records
|
|
131
130
|
const emailDnsRecords = await this.generateEmailDnsRecords();
|
|
132
131
|
|
|
133
|
-
//
|
|
134
|
-
await this.initializeDkimForEmailDomains();
|
|
135
|
-
|
|
136
|
-
// Generate DKIM records directly from smartmta.
|
|
132
|
+
// Load caller-managed DKIM records without generating or rotating keys.
|
|
137
133
|
const dkimRecords = await this.loadDkimRecords();
|
|
138
134
|
|
|
139
135
|
// Combine all records: authoritative, email, DKIM, and user-defined
|
|
@@ -367,7 +363,10 @@ export class DnsServerRuntime {
|
|
|
367
363
|
}
|
|
368
364
|
const selector = domainConfig.dkim?.selector || 'default';
|
|
369
365
|
try {
|
|
370
|
-
const dkimRecord = await this.dcRouterRef.emailServer.dkimCreator.
|
|
366
|
+
const dkimRecord = await this.dcRouterRef.emailServer.dkimCreator.getDNSRecordForSelector(
|
|
367
|
+
domainConfig.domain,
|
|
368
|
+
selector,
|
|
369
|
+
);
|
|
371
370
|
records.push({
|
|
372
371
|
name: dkimRecord.name,
|
|
373
372
|
type: 'TXT',
|
|
@@ -375,7 +374,10 @@ export class DnsServerRuntime {
|
|
|
375
374
|
ttl: domainConfig.dns?.internal?.ttl || 3600,
|
|
376
375
|
});
|
|
377
376
|
} catch (error: unknown) {
|
|
378
|
-
logger.log(
|
|
377
|
+
logger.log(
|
|
378
|
+
'error',
|
|
379
|
+
`Caller-managed DKIM material is unavailable for ${selector}._domainkey.${domainConfig.domain}: ${(error as Error).message}`,
|
|
380
|
+
);
|
|
379
381
|
}
|
|
380
382
|
}
|
|
381
383
|
|
|
@@ -389,44 +391,6 @@ export class DnsServerRuntime {
|
|
|
389
391
|
);
|
|
390
392
|
}
|
|
391
393
|
|
|
392
|
-
private async initializeDkimForEmailDomains(): Promise<void> {
|
|
393
|
-
const options = this.dcRouterRef.options;
|
|
394
|
-
if (!options.emailConfig?.domains || !this.dcRouterRef.emailServer) {
|
|
395
|
-
return;
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
logger.log('info', 'Initializing DKIM keys for email domains...');
|
|
399
|
-
|
|
400
|
-
// Get DKIMCreator instance from email server (public in smartmta)
|
|
401
|
-
const dkimCreator = this.dcRouterRef.emailServer.dkimCreator;
|
|
402
|
-
if (!dkimCreator) {
|
|
403
|
-
logger.log('warn', 'DKIMCreator not available, skipping DKIM initialization');
|
|
404
|
-
return;
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
// Ensure necessary directories exist
|
|
408
|
-
paths.ensureDataDirectories(this.dcRouterRef.resolvedPaths);
|
|
409
|
-
|
|
410
|
-
// Generate DKIM keys for each internal-dns email domain using the configured selector.
|
|
411
|
-
for (const domainConfig of options.emailConfig.domains) {
|
|
412
|
-
if (domainConfig.dnsMode !== 'internal-dns') {
|
|
413
|
-
continue;
|
|
414
|
-
}
|
|
415
|
-
try {
|
|
416
|
-
await dkimCreator.handleDKIMKeysForSelector(
|
|
417
|
-
domainConfig.domain,
|
|
418
|
-
domainConfig.dkim?.selector || 'default',
|
|
419
|
-
domainConfig.dkim?.keySize || 2048,
|
|
420
|
-
);
|
|
421
|
-
logger.log('info', `DKIM keys initialized for ${domainConfig.domain}`);
|
|
422
|
-
} catch (error: unknown) {
|
|
423
|
-
logger.log('error', `Failed to initialize DKIM for ${domainConfig.domain}: ${(error as Error).message}`);
|
|
424
|
-
}
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
logger.log('info', 'DKIM initialization complete');
|
|
428
|
-
}
|
|
429
|
-
|
|
430
394
|
private async generateAuthoritativeRecords(): Promise<TDnsRecordSeed[]> {
|
|
431
395
|
const options = this.dcRouterRef.options;
|
|
432
396
|
const records: TDnsRecordSeed[] = [];
|
package/ts/dns/index.ts
CHANGED