@serve.zone/dcrouter 17.10.2 → 18.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deno.json +1 -1
- package/dist_serve/bundle.js +360 -360
- package/dist_ts/00_commitinfo_data.js +2 -2
- package/dist_ts/acme/acme-failure-classification.d.ts +64 -0
- package/dist_ts/acme/acme-failure-classification.js +114 -0
- package/dist_ts/acme/classes.smartacme-lifecycle.d.ts +42 -0
- package/dist_ts/acme/classes.smartacme-lifecycle.js +75 -4
- package/dist_ts/acme/index.d.ts +1 -0
- package/dist_ts/acme/index.js +2 -1
- package/dist_ts/classes.dcrouter.d.ts +33 -9
- package/dist_ts/classes.dcrouter.js +145 -11
- package/dist_ts/config/classes.route-config-manager.d.ts +56 -0
- package/dist_ts/config/classes.route-config-manager.js +161 -1
- package/dist_ts/db/documents/classes.dns-authority.doc.d.ts +30 -0
- package/dist_ts/db/documents/classes.dns-authority.doc.js +108 -0
- package/dist_ts/db/documents/index.d.ts +1 -0
- package/dist_ts/db/documents/index.js +2 -1
- package/dist_ts/dns/classes.dns-server-runtime.d.ts +109 -1
- package/dist_ts/dns/classes.dns-server-runtime.js +212 -34
- package/dist_ts/dns/domain-ownership.d.ts +111 -0
- package/dist_ts/dns/domain-ownership.js +152 -0
- package/dist_ts/dns/index.d.ts +2 -0
- package/dist_ts/dns/index.js +3 -1
- package/dist_ts/dns/manager.dns-authority.d.ts +143 -0
- package/dist_ts/dns/manager.dns-authority.js +481 -0
- package/dist_ts/dns/manager.dns.d.ts +121 -12
- package/dist_ts/dns/manager.dns.js +298 -28
- package/dist_ts/email/classes.accepted-email-spool.d.ts +15 -0
- package/dist_ts/email/classes.accepted-email-spool.js +102 -23
- package/dist_ts/email/classes.smartmta-blob-storage-manager.js +12 -3
- package/dist_ts/errors/error.codes.d.ts +4 -0
- package/dist_ts/errors/error.codes.js +5 -1
- package/dist_ts/opsserver/classes.opsserver.d.ts +1 -0
- package/dist_ts/opsserver/classes.opsserver.js +3 -1
- package/dist_ts/opsserver/handlers/acme-config.handler.js +6 -1
- package/dist_ts/opsserver/handlers/certificate.handler.d.ts +16 -0
- package/dist_ts/opsserver/handlers/certificate.handler.js +96 -10
- package/dist_ts/opsserver/handlers/config.handler.js +4 -2
- package/dist_ts/opsserver/handlers/dns-authority.handler.d.ts +20 -0
- package/dist_ts/opsserver/handlers/dns-authority.handler.js +108 -0
- package/dist_ts/opsserver/handlers/dns-provider.handler.js +5 -1
- package/dist_ts/opsserver/handlers/domain.handler.js +9 -1
- package/dist_ts/opsserver/handlers/gatewayclient.handler.js +2 -2
- package/dist_ts/opsserver/handlers/index.d.ts +1 -0
- package/dist_ts/opsserver/handlers/index.js +2 -1
- package/dist_ts_interfaces/data/dns-authority.d.ts +98 -0
- package/dist_ts_interfaces/data/dns-authority.js +26 -0
- package/dist_ts_interfaces/data/index.d.ts +1 -0
- package/dist_ts_interfaces/data/index.js +2 -1
- package/dist_ts_interfaces/data/route-management.d.ts +9 -2
- package/dist_ts_interfaces/data/route-management.js +3 -1
- package/dist_ts_interfaces/requests/certificate.d.ts +23 -0
- package/dist_ts_interfaces/requests/certificate.js +1 -1
- package/dist_ts_interfaces/requests/dns-authority.d.ts +80 -0
- package/dist_ts_interfaces/requests/dns-authority.js +3 -0
- package/dist_ts_interfaces/requests/index.d.ts +1 -0
- package/dist_ts_interfaces/requests/index.js +2 -1
- package/dist_ts_oci_container/index.js +9 -4
- package/dist_ts_web/00_commitinfo_data.js +2 -2
- package/package.json +1 -1
- package/readme.hints.md +412 -0
- package/readme.md +48 -6
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/acme/acme-failure-classification.ts +201 -0
- package/ts/acme/classes.smartacme-lifecycle.ts +104 -3
- package/ts/acme/index.ts +1 -0
- package/ts/classes.dcrouter.ts +193 -23
- package/ts/config/classes.route-config-manager.ts +197 -0
- package/ts/db/documents/classes.dns-authority.doc.ts +49 -0
- package/ts/db/documents/index.ts +1 -0
- package/ts/dns/classes.dns-server-runtime.ts +257 -38
- package/ts/dns/domain-ownership.ts +272 -0
- package/ts/dns/index.ts +2 -0
- package/ts/dns/manager.dns-authority.ts +558 -0
- package/ts/dns/manager.dns.ts +373 -27
- package/ts/email/classes.accepted-email-spool.ts +98 -21
- package/ts/email/classes.smartmta-blob-storage-manager.ts +10 -2
- package/ts/errors/error.codes.ts +4 -0
- package/ts/opsserver/classes.opsserver.ts +2 -0
- package/ts/opsserver/handlers/acme-config.handler.ts +7 -0
- package/ts/opsserver/handlers/certificate.handler.ts +103 -8
- package/ts/opsserver/handlers/config.handler.ts +3 -1
- package/ts/opsserver/handlers/dns-authority.handler.ts +142 -0
- package/ts/opsserver/handlers/dns-provider.handler.ts +6 -0
- package/ts/opsserver/handlers/domain.handler.ts +12 -0
- package/ts/opsserver/handlers/gatewayclient.handler.ts +1 -1
- package/ts/opsserver/handlers/index.ts +1 -0
- package/ts/readme.md +1 -1
- package/ts_web/00_commitinfo_data.ts +1 -1
|
@@ -77,6 +77,19 @@ const ACCEPTED_EMAIL_STOP_DRAIN_TIMEOUT_MS = 30_000;
|
|
|
77
77
|
/** Retention for catch-all stored inbound mail (30 days). */
|
|
78
78
|
const INBOUND_STORE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
|
|
79
79
|
|
|
80
|
+
/**
|
|
81
|
+
* Permanent per-email storage failure: the raw RFC822 payload of an accepted
|
|
82
|
+
* email is unrecoverable, so delivery can never succeed. The spool marks such
|
|
83
|
+
* rows failed instead of retrying them, so one lost blob cannot wedge the
|
|
84
|
+
* whole queue.
|
|
85
|
+
*/
|
|
86
|
+
export class AcceptedEmailRawMessageMissingError extends Error {
|
|
87
|
+
constructor(messageArg: string) {
|
|
88
|
+
super(messageArg);
|
|
89
|
+
this.name = 'AcceptedEmailRawMessageMissingError';
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
80
93
|
export type TSmartMtaQueueItemLike = {
|
|
81
94
|
id?: string;
|
|
82
95
|
processingResult?: {
|
|
@@ -690,32 +703,55 @@ export class AcceptedEmailSpool {
|
|
|
690
703
|
for (const item of (emailServerArg as TSmartMtaQueueReader).getQueueItems?.() || []) {
|
|
691
704
|
const cachedEmailId = this.getCachedEmailIdFromQueueItem(item);
|
|
692
705
|
if (!cachedEmailId || !item.smtpTransactions?.length) continue;
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
706
|
+
try {
|
|
707
|
+
await this.runCachedEmailUpdate(cachedEmailId, async () => {
|
|
708
|
+
const cachedEmail = await CachedEmail.findById(cachedEmailId);
|
|
709
|
+
if (!cachedEmail) return;
|
|
710
|
+
for (const transaction of item.smtpTransactions || []) {
|
|
711
|
+
this.appendSmtpTransaction(cachedEmail, transaction);
|
|
712
|
+
}
|
|
713
|
+
await cachedEmail.save();
|
|
714
|
+
});
|
|
715
|
+
} catch (error: unknown) {
|
|
716
|
+
// History reconciliation runs during email-server startup; one row that
|
|
717
|
+
// cannot be persisted must not abort reconciliation or startup.
|
|
718
|
+
logger.log('warn', `Unable to reconcile SMTP transaction history for accepted email ${cachedEmailId}: ${(error as Error).message}`);
|
|
719
|
+
}
|
|
701
720
|
}
|
|
702
721
|
}
|
|
703
722
|
|
|
704
723
|
/** Requeue emails left in 'queued' state by a previous process as pending. */
|
|
705
724
|
public async recoverQueuedEmails(): Promise<void> {
|
|
725
|
+
// Rows whose recovery save failed stay 'queued' and are re-served by
|
|
726
|
+
// findQueuedForRecovery; track them so they are neither retried in a loop
|
|
727
|
+
// within this run nor able to wedge recovery when they fill a whole batch.
|
|
728
|
+
const unrecoverableIds = new Set<string>();
|
|
706
729
|
while (true) {
|
|
707
730
|
const queuedEmails = await CachedEmail.findQueuedForRecovery(ACCEPTED_EMAIL_SPOOL_BATCH_SIZE);
|
|
708
|
-
|
|
731
|
+
const recoverableEmails = queuedEmails.filter((queuedEmail) => !unrecoverableIds.has(queuedEmail.id));
|
|
732
|
+
if (recoverableEmails.length === 0) {
|
|
709
733
|
return;
|
|
710
734
|
}
|
|
711
|
-
for (const queuedEmail of
|
|
735
|
+
for (const queuedEmail of recoverableEmails) {
|
|
712
736
|
if (this.isCachedEmailTerminal(queuedEmail)) {
|
|
713
737
|
continue;
|
|
714
738
|
}
|
|
715
|
-
queuedEmail.status
|
|
716
|
-
queuedEmail.nextAttempt
|
|
717
|
-
|
|
718
|
-
|
|
739
|
+
const previousStatus = queuedEmail.status;
|
|
740
|
+
const previousNextAttempt = queuedEmail.nextAttempt;
|
|
741
|
+
try {
|
|
742
|
+
queuedEmail.status = 'pending';
|
|
743
|
+
queuedEmail.nextAttempt = new Date();
|
|
744
|
+
await queuedEmail.save();
|
|
745
|
+
await this.notifyEmailQueuePersisted(queuedEmail, 'email-queue-recovered');
|
|
746
|
+
} catch (error: unknown) {
|
|
747
|
+
// Recovery runs during email-server startup; a row that cannot be
|
|
748
|
+
// persisted stays 'queued' for the next startup and must not abort
|
|
749
|
+
// recovery of the remaining rows.
|
|
750
|
+
queuedEmail.status = previousStatus;
|
|
751
|
+
queuedEmail.nextAttempt = previousNextAttempt;
|
|
752
|
+
unrecoverableIds.add(queuedEmail.id);
|
|
753
|
+
logger.log('warn', `Unable to recover queued accepted email ${queuedEmail.id}: ${(error as Error).message}`);
|
|
754
|
+
}
|
|
719
755
|
}
|
|
720
756
|
if (queuedEmails.length < ACCEPTED_EMAIL_SPOOL_BATCH_SIZE) {
|
|
721
757
|
return;
|
|
@@ -735,18 +771,54 @@ export class AcceptedEmailSpool {
|
|
|
735
771
|
if (this.stopping || this.dcRouterRef.emailServer !== emailServer) {
|
|
736
772
|
break;
|
|
737
773
|
}
|
|
738
|
-
|
|
739
|
-
|
|
774
|
+
try {
|
|
775
|
+
if (await this.postponeLiveSmartMtaOwnedEmail(cachedEmail, emailServer)) {
|
|
776
|
+
continue;
|
|
777
|
+
}
|
|
778
|
+
const session = this.buildCachedEmailSession(cachedEmail);
|
|
779
|
+
const rawMessage = await this.readRawMessage(cachedEmail);
|
|
780
|
+
await this.processAcceptedCachedEmail(cachedEmail, rawMessage, session, emailServer);
|
|
781
|
+
} catch (error: unknown) {
|
|
782
|
+
// One poisoned row must never stall the rest of the batch: record the
|
|
783
|
+
// failure on that row alone and keep processing the remaining emails.
|
|
784
|
+
await this.handleSpoolItemFailure(cachedEmail, error);
|
|
740
785
|
}
|
|
741
|
-
const session = this.buildCachedEmailSession(cachedEmail);
|
|
742
|
-
const rawMessage = await this.readRawMessage(cachedEmail);
|
|
743
|
-
await this.processAcceptedCachedEmail(cachedEmail, rawMessage, session, emailServer);
|
|
744
786
|
}
|
|
745
787
|
} finally {
|
|
746
788
|
this.processing = false;
|
|
747
789
|
}
|
|
748
790
|
}
|
|
749
791
|
|
|
792
|
+
/**
|
|
793
|
+
* Record a per-email spool failure without aborting the batch: permanent
|
|
794
|
+
* raw-message losses are marked failed, everything else is deferred through
|
|
795
|
+
* the regular retry mechanism (which terminates at maxAttempts).
|
|
796
|
+
*/
|
|
797
|
+
private async handleSpoolItemFailure(cachedEmail: CachedEmail, errorArg: unknown): Promise<void> {
|
|
798
|
+
const message = (errorArg as Error).message;
|
|
799
|
+
try {
|
|
800
|
+
const currentCachedEmail = await CachedEmail.findById(cachedEmail.id) || cachedEmail;
|
|
801
|
+
if (this.isCachedEmailTerminal(currentCachedEmail)) {
|
|
802
|
+
return;
|
|
803
|
+
}
|
|
804
|
+
if (errorArg instanceof AcceptedEmailRawMessageMissingError) {
|
|
805
|
+
currentCachedEmail.markFailed(message);
|
|
806
|
+
await currentCachedEmail.save();
|
|
807
|
+
await this.notifyEmailQueuePersisted(currentCachedEmail, 'email-raw-message-missing');
|
|
808
|
+
logger.log('error', `Accepted email ${currentCachedEmail.id} failed permanently: ${message}`);
|
|
809
|
+
return;
|
|
810
|
+
}
|
|
811
|
+
currentCachedEmail.scheduleRetry(ACCEPTED_EMAIL_RETRY_DELAY_MS);
|
|
812
|
+
currentCachedEmail.lastError = message;
|
|
813
|
+
await currentCachedEmail.save();
|
|
814
|
+
await this.notifyEmailQueuePersisted(currentCachedEmail, 'email-spool-deferred');
|
|
815
|
+
logger.log('warn', `Accepted email ${currentCachedEmail.id} deferred after spool processing failure: ${message}`);
|
|
816
|
+
} catch (persistError: unknown) {
|
|
817
|
+
// Even failing to persist the failure state must not abort the batch.
|
|
818
|
+
logger.log('error', `Unable to persist spool failure state for accepted email ${cachedEmail.id}: ${(persistError as Error).message} (original error: ${message})`);
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
|
|
750
822
|
private async processAcceptedCachedEmail(
|
|
751
823
|
cachedEmail: CachedEmail,
|
|
752
824
|
emailData: Email | plugins.buffer.Buffer,
|
|
@@ -1010,11 +1082,16 @@ export class AcceptedEmailSpool {
|
|
|
1010
1082
|
}
|
|
1011
1083
|
const rawMessage = await blobStorage.get(cachedEmailArg.rawContentObjectKey);
|
|
1012
1084
|
if (!rawMessage) {
|
|
1013
|
-
throw new
|
|
1085
|
+
throw new AcceptedEmailRawMessageMissingError(`Raw RFC822 object is missing for accepted email ${cachedEmailArg.id}`);
|
|
1014
1086
|
}
|
|
1015
1087
|
return plugins.buffer.Buffer.from(rawMessage);
|
|
1016
1088
|
}
|
|
1017
|
-
|
|
1089
|
+
if (cachedEmailArg.rawContent) {
|
|
1090
|
+
// Legacy rows persisted before SmartBucket-backed storage keep their
|
|
1091
|
+
// payload inline; serve it rather than declaring the payload lost.
|
|
1092
|
+
return plugins.buffer.Buffer.from(cachedEmailArg.rawContent, 'utf8');
|
|
1093
|
+
}
|
|
1094
|
+
throw new AcceptedEmailRawMessageMissingError(`Accepted email ${cachedEmailArg.id} has no raw RFC822 payload`);
|
|
1018
1095
|
}
|
|
1019
1096
|
|
|
1020
1097
|
public async deleteRawMessage(cachedEmailArg: CachedEmail): Promise<void> {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as plugins from '../plugins.js';
|
|
2
|
+
import { logger } from '../logger.js';
|
|
2
3
|
import type { IBlobStorageManager } from '@push.rocks/smartmta';
|
|
3
4
|
|
|
4
5
|
const SMARTMTA_BLOB_NAMESPACES = [
|
|
@@ -119,11 +120,18 @@ export class SmartMtaBlobStorageManager implements IBlobStorageManager {
|
|
|
119
120
|
const objectPrefix = this.toObjectKey(prefix);
|
|
120
121
|
const keys: string[] = [];
|
|
121
122
|
for await (const objectKey of this.bucketRef.listAllObjects(objectPrefix)) {
|
|
123
|
+
// One stray or malformed object under the prefix must not abort the
|
|
124
|
+
// whole listing (SmartMTA queue recovery lists at startup); skip it.
|
|
122
125
|
if (!objectKey.startsWith(this.objectPrefix)) {
|
|
123
|
-
|
|
126
|
+
logger.log('warn', `Skipping SmartBucket object outside the configured prefix during SmartMTA blob listing: ${objectKey}`);
|
|
127
|
+
continue;
|
|
124
128
|
}
|
|
125
129
|
const key = `/${objectKey.slice(this.objectPrefix.length)}`;
|
|
126
|
-
|
|
130
|
+
try {
|
|
131
|
+
keys.push(normalizeBlobKey(key));
|
|
132
|
+
} catch (error: unknown) {
|
|
133
|
+
logger.log('warn', `Skipping unnormalizable SmartBucket object key during SmartMTA blob listing: ${objectKey} (${(error as Error).message})`);
|
|
134
|
+
}
|
|
127
135
|
}
|
|
128
136
|
return keys.sort();
|
|
129
137
|
}
|
package/ts/errors/error.codes.ts
CHANGED
|
@@ -80,6 +80,10 @@ export const DCR_ROUTING_ERROR = 'DCR_ROUTING_ERROR';
|
|
|
80
80
|
export const DCR_CONFIGURATION_ERROR = 'DCR_CONFIGURATION_ERROR';
|
|
81
81
|
export const DCR_PROXY_ERROR = 'DCR_PROXY_ERROR';
|
|
82
82
|
export const DCR_DOMAIN_ERROR = 'DCR_DOMAIN_ERROR';
|
|
83
|
+
/** A certificate requirement or authoritative DNS zone was requested for a domain we cannot prove we own. */
|
|
84
|
+
export const DCR_DOMAIN_OWNERSHIP_UNVERIFIED = 'DCR_DOMAIN_OWNERSHIP_UNVERIFIED';
|
|
85
|
+
/** ACME failed for a configuration cause that no amount of retrying can resolve. */
|
|
86
|
+
export const DCR_ACME_PERMANENT_FAILURE = 'DCR_ACME_PERMANENT_FAILURE';
|
|
83
87
|
|
|
84
88
|
// SMS service errors (SMS_*)
|
|
85
89
|
export const SMS_SERVICE_ERROR = 'SMS_SERVICE_ERROR';
|
|
@@ -37,6 +37,7 @@ export class OpsServer {
|
|
|
37
37
|
private usersHandler!: handlers.UsersHandler;
|
|
38
38
|
private dnsProviderHandler!: handlers.DnsProviderHandler;
|
|
39
39
|
private domainHandler!: handlers.DomainHandler;
|
|
40
|
+
private dnsAuthorityHandler!: handlers.DnsAuthorityHandler;
|
|
40
41
|
private dnsRecordHandler!: handlers.DnsRecordHandler;
|
|
41
42
|
private acmeConfigHandler!: handlers.AcmeConfigHandler;
|
|
42
43
|
private emailDomainHandler!: handlers.EmailDomainHandler;
|
|
@@ -104,6 +105,7 @@ export class OpsServer {
|
|
|
104
105
|
this.usersHandler = new handlers.UsersHandler(this);
|
|
105
106
|
this.dnsProviderHandler = new handlers.DnsProviderHandler(this);
|
|
106
107
|
this.domainHandler = new handlers.DomainHandler(this);
|
|
108
|
+
this.dnsAuthorityHandler = new handlers.DnsAuthorityHandler(this);
|
|
107
109
|
this.dnsRecordHandler = new handlers.DnsRecordHandler(this);
|
|
108
110
|
this.acmeConfigHandler = new handlers.AcmeConfigHandler(this);
|
|
109
111
|
this.emailDomainHandler = new handlers.EmailDomainHandler(this);
|
|
@@ -66,6 +66,13 @@ export class AcmeConfigHandler {
|
|
|
66
66
|
},
|
|
67
67
|
userId,
|
|
68
68
|
);
|
|
69
|
+
// The SmartAcme startup budget does not re-arm on a timer, so a
|
|
70
|
+
// corrected ACME configuration must re-arm it explicitly. Otherwise a
|
|
71
|
+
// permanently-failed or exhausted provider stays down until an
|
|
72
|
+
// unrelated SmartProxy rebuild or a restart (30–60 s of outage).
|
|
73
|
+
this.opsServerRef.dcRouterRef.smartAcmeLifecycle?.rearm(
|
|
74
|
+
'ACME configuration updated',
|
|
75
|
+
);
|
|
69
76
|
return { success: true, config: updated };
|
|
70
77
|
} catch (err: unknown) {
|
|
71
78
|
return { success: false, message: (err as Error).message };
|
|
@@ -266,22 +266,35 @@ export class CertificateHandler {
|
|
|
266
266
|
}
|
|
267
267
|
}
|
|
268
268
|
|
|
269
|
-
//
|
|
270
|
-
|
|
269
|
+
// What the running proxy actually serves. Always queried, not only as a
|
|
270
|
+
// fallback: the served state is an independent fact from the stored one and
|
|
271
|
+
// the two must be comparable.
|
|
272
|
+
let served: interfaces.requests.ICertificateServedState | undefined;
|
|
273
|
+
if (info.routeNames.length > 0) {
|
|
271
274
|
try {
|
|
272
275
|
const rustStatus = await smartProxy.getCertificateStatus(info.routeNames[0]);
|
|
273
276
|
if (rustStatus) {
|
|
274
|
-
|
|
275
|
-
expiryDate
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
277
|
+
served = {
|
|
278
|
+
expiryDate: rustStatus.expiresAt > 0
|
|
279
|
+
? new Date(rustStatus.expiresAt).toISOString()
|
|
280
|
+
: undefined,
|
|
281
|
+
isValid: rustStatus.isValid,
|
|
282
|
+
source: rustStatus.source || undefined,
|
|
283
|
+
};
|
|
279
284
|
}
|
|
280
285
|
} catch {
|
|
281
|
-
// Rust bridge may not support this command yet —
|
|
286
|
+
// Rust bridge may not support this command yet — leave served unknown.
|
|
282
287
|
}
|
|
283
288
|
}
|
|
284
289
|
|
|
290
|
+
// With no event data, the served state is also the best available view of
|
|
291
|
+
// the issued one.
|
|
292
|
+
if (status === 'unknown' && served) {
|
|
293
|
+
if (served.expiryDate) expiryDate = served.expiryDate;
|
|
294
|
+
if (served.source) issuer = served.source;
|
|
295
|
+
status = served.isValid ? 'valid' : 'expired';
|
|
296
|
+
}
|
|
297
|
+
|
|
285
298
|
// Check persisted cert data from smartdata document classes
|
|
286
299
|
if (status === 'unknown') {
|
|
287
300
|
const cleanDomain = domain.replace(/^\*\.?/, '');
|
|
@@ -352,6 +365,23 @@ export class CertificateHandler {
|
|
|
352
365
|
error = error || backoffInfo.lastError;
|
|
353
366
|
}
|
|
354
367
|
|
|
368
|
+
// Issued is not served. A certificate valid in storage while the engine
|
|
369
|
+
// still presents an invalid or expired one is a failure, however healthy
|
|
370
|
+
// the issuance path looks — this is exactly what hid a 21-day outage.
|
|
371
|
+
const servedMismatch = Boolean(
|
|
372
|
+
served
|
|
373
|
+
&& !served.isValid
|
|
374
|
+
&& (status === 'valid' || status === 'expiring'),
|
|
375
|
+
);
|
|
376
|
+
if (servedMismatch) {
|
|
377
|
+
status = 'failed';
|
|
378
|
+
error = error
|
|
379
|
+
|| `A valid certificate is stored for '${domain}'`
|
|
380
|
+
+ `${expiryDate ? ` (expires ${expiryDate})` : ''}, but the running proxy is serving an invalid one`
|
|
381
|
+
+ `${served?.expiryDate ? ` (expired ${served.expiryDate})` : ''}. `
|
|
382
|
+
+ 'The issued certificate has not reached the engine.';
|
|
383
|
+
}
|
|
384
|
+
|
|
355
385
|
certificates.push({
|
|
356
386
|
domain,
|
|
357
387
|
routeNames: info.routeNames,
|
|
@@ -363,6 +393,8 @@ export class CertificateHandler {
|
|
|
363
393
|
issuedAt,
|
|
364
394
|
error,
|
|
365
395
|
canReprovision: info.canReprovision,
|
|
396
|
+
served,
|
|
397
|
+
...(servedMismatch ? { servedMismatch: true } : {}),
|
|
366
398
|
backoffInfo,
|
|
367
399
|
});
|
|
368
400
|
}
|
|
@@ -419,6 +451,21 @@ export class CertificateHandler {
|
|
|
419
451
|
return { success: false, message: `No routes found for domain '${domain}'` };
|
|
420
452
|
}
|
|
421
453
|
|
|
454
|
+
// Refuse before triggering an order that cannot succeed. forceRenew is an
|
|
455
|
+
// operator override for backoff, not for missing ownership: without a zone we
|
|
456
|
+
// can prove is ours there is nowhere to place the DNS-01 challenge record.
|
|
457
|
+
if (dcRouter.dnsManager) {
|
|
458
|
+
const ownership = await dcRouter.dnsManager.resolveDomainOwnership(domain);
|
|
459
|
+
if (!ownership.verified) {
|
|
460
|
+
return {
|
|
461
|
+
success: false,
|
|
462
|
+
message:
|
|
463
|
+
`Cannot provision a certificate for '${domain}': domain ownership is unverified `
|
|
464
|
+
+ `(${ownership.reason}) — ${ownership.detail}`,
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
|
|
422
469
|
// Respect existing backoff unless explicitly overridden. Automated
|
|
423
470
|
// callers (gateway clients retrying cert fetches) used to reset the
|
|
424
471
|
// backoff and trigger a route re-apply on every request, producing a
|
|
@@ -488,12 +535,60 @@ export class CertificateHandler {
|
|
|
488
535
|
// Fallback when DB is disabled and there is no RouteConfigManager
|
|
489
536
|
await smartProxy.updateRoutes(smartProxy.routeManager.getRoutes());
|
|
490
537
|
}
|
|
538
|
+
const propagation = await this.describeCertPropagationGap(domain, routeNames);
|
|
539
|
+
if (propagation) {
|
|
540
|
+
return { success: false, message: propagation };
|
|
541
|
+
}
|
|
491
542
|
return { success: true, message: forceRenew ? `Certificate force-renewed for domain '${domain}'` : `Certificate reprovisioning triggered for domain '${domain}'` };
|
|
492
543
|
} catch (err: unknown) {
|
|
493
544
|
return { success: false, message: (err as Error).message || `Failed to reprovision certificate for ${domain}` };
|
|
494
545
|
}
|
|
495
546
|
}
|
|
496
547
|
|
|
548
|
+
/**
|
|
549
|
+
* After triggering the apply pipeline, check whether the certificate the proxy
|
|
550
|
+
* is actually serving is still expired. If it is, the reprovision did not
|
|
551
|
+
* propagate and reporting success would be a lie.
|
|
552
|
+
*
|
|
553
|
+
* The known cause is SmartProxy's own per-domain provisioning cooldown
|
|
554
|
+
* (`certProvisionFailureCooldownMs`, 30 min default). It is private in-memory
|
|
555
|
+
* state with no invalidation path: a provisioning sweep skips a cooling-down
|
|
556
|
+
* domain, and because the sweep returns before its summary log when every
|
|
557
|
+
* domain is skipped, the skip is not logged either. So a valid certificate can
|
|
558
|
+
* sit in storage while the proxy keeps serving the expired one, silently.
|
|
559
|
+
*
|
|
560
|
+
* Returns undefined when propagation looks fine or cannot be determined —
|
|
561
|
+
* `getCertificateStatus` goes through the Rust bridge and may be unavailable.
|
|
562
|
+
*/
|
|
563
|
+
private async describeCertPropagationGap(
|
|
564
|
+
domain: string,
|
|
565
|
+
routeNames: string[],
|
|
566
|
+
): Promise<string | undefined> {
|
|
567
|
+
const smartProxy = this.opsServerRef.dcRouterRef.smartProxy;
|
|
568
|
+
if (!smartProxy || routeNames.length === 0) {
|
|
569
|
+
return undefined;
|
|
570
|
+
}
|
|
571
|
+
let status: Awaited<ReturnType<typeof smartProxy.getCertificateStatus>>;
|
|
572
|
+
try {
|
|
573
|
+
status = await smartProxy.getCertificateStatus(routeNames[0]);
|
|
574
|
+
} catch {
|
|
575
|
+
// Bridge command unsupported — cannot determine, so do not claim a gap.
|
|
576
|
+
return undefined;
|
|
577
|
+
}
|
|
578
|
+
if (!status || status.isValid || !status.expiresAt || status.expiresAt <= 0) {
|
|
579
|
+
return undefined;
|
|
580
|
+
}
|
|
581
|
+
const servedUntil = new Date(status.expiresAt).toISOString();
|
|
582
|
+
const message =
|
|
583
|
+
`Certificate for '${domain}' was issued and stored, but route '${routeNames[0]}' is still serving a `
|
|
584
|
+
+ `certificate that expired at ${servedUntil}. The new certificate did not reach the running proxy: `
|
|
585
|
+
+ 'SmartProxy keeps a private per-domain provisioning cooldown that a sweep cannot clear while it is '
|
|
586
|
+
+ 'active, and it exposes no per-domain invalidation or hot-load API. Toggling this single route off and '
|
|
587
|
+
+ 'back on is the only supported way to force a fresh attempt without restarting dcrouter.';
|
|
588
|
+
logger.log('error', message, { domain, routeName: routeNames[0] });
|
|
589
|
+
return message;
|
|
590
|
+
}
|
|
591
|
+
|
|
497
592
|
/**
|
|
498
593
|
* After a force-renew, walk every route in the smartproxy that resolves to
|
|
499
594
|
* the same cert identity as `forcedDomain` and write the freshly-issued cert
|
|
@@ -137,7 +137,9 @@ export class ConfigHandler {
|
|
|
137
137
|
enabled: !!dcRouter.dnsServer,
|
|
138
138
|
port: 53,
|
|
139
139
|
nsDomains: opts.dnsNsDomains || [],
|
|
140
|
-
|
|
140
|
+
// The delegation-verified authority set. Deployment configuration no
|
|
141
|
+
// longer contributes: this is database state, and it is the only state.
|
|
142
|
+
scopes: dcRouter.dnsAuthorityManager?.getEffectiveZoneNames() || [],
|
|
141
143
|
recordCount: dnsRecords.length,
|
|
142
144
|
records: dnsRecords,
|
|
143
145
|
dnsChallenge: dnsChallengeEnabled,
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import * as plugins from '../../plugins.js';
|
|
2
|
+
import type { OpsServer } from '../classes.opsserver.js';
|
|
3
|
+
import * as interfaces from '../../../ts_interfaces/index.js';
|
|
4
|
+
import { requireOpsAuth } from '../helpers/auth.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* DNS authority handlers.
|
|
8
|
+
*
|
|
9
|
+
* Adding a zone here is an authority claim — after it succeeds dcrouter answers
|
|
10
|
+
* for that zone with the `aa` flag on a public port and will request
|
|
11
|
+
* certificates for it. So it carries the same posture as the other DNS write
|
|
12
|
+
* surfaces (admin identity or a `:write`-scoped token) *and* the claim itself is
|
|
13
|
+
* gated on a delegation proof the caller cannot forge. Authorization decides who
|
|
14
|
+
* may ask; the probe decides whether the answer is yes.
|
|
15
|
+
*/
|
|
16
|
+
export class DnsAuthorityHandler {
|
|
17
|
+
public typedrouter = new plugins.typedrequest.TypedRouter();
|
|
18
|
+
|
|
19
|
+
constructor(private opsServerRef: OpsServer) {
|
|
20
|
+
this.opsServerRef.typedrouter.addTypedRouter(this.typedrouter);
|
|
21
|
+
this.registerHandlers();
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
private async requireAuth(
|
|
25
|
+
request: { identity?: interfaces.data.IIdentity; apiToken?: string },
|
|
26
|
+
requiredScope?: interfaces.data.TApiTokenScope,
|
|
27
|
+
): Promise<string> {
|
|
28
|
+
const auth = await requireOpsAuth(this.opsServerRef, request, {
|
|
29
|
+
scope: requiredScope,
|
|
30
|
+
requireAdminIdentity: requiredScope?.endsWith(':write'),
|
|
31
|
+
});
|
|
32
|
+
return auth.userId;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
private registerHandlers(): void {
|
|
36
|
+
// Read the authority set: delegation-verified zones, and nothing else.
|
|
37
|
+
this.typedrouter.addTypedHandler(
|
|
38
|
+
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetDnsAuthority>(
|
|
39
|
+
'getDnsAuthority',
|
|
40
|
+
async (dataArg) => {
|
|
41
|
+
await this.requireAuth(dataArg, 'dns-authority:read');
|
|
42
|
+
const manager = this.opsServerRef.dcRouterRef.dnsAuthorityManager;
|
|
43
|
+
if (!manager) {
|
|
44
|
+
// No manager means the set was never loaded. Reported as
|
|
45
|
+
// 'unavailable' rather than as an empty set, so a reader cannot
|
|
46
|
+
// mistake "we could not look" for "there is nothing".
|
|
47
|
+
return {
|
|
48
|
+
settings: {
|
|
49
|
+
zones: [],
|
|
50
|
+
state: 'unavailable' as const,
|
|
51
|
+
expectedNameservers: [],
|
|
52
|
+
updatedAt: 0,
|
|
53
|
+
updatedBy: '',
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
return { settings: manager.getSettings() };
|
|
58
|
+
},
|
|
59
|
+
),
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
// Probe a zone's delegation without changing anything
|
|
63
|
+
this.typedrouter.addTypedHandler(
|
|
64
|
+
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_ProbeDnsAuthorityZone>(
|
|
65
|
+
'probeDnsAuthorityZone',
|
|
66
|
+
async (dataArg) => {
|
|
67
|
+
await this.requireAuth(dataArg, 'dns-authority:read');
|
|
68
|
+
const manager = this.opsServerRef.dcRouterRef.dnsAuthorityManager;
|
|
69
|
+
if (!manager) {
|
|
70
|
+
return {
|
|
71
|
+
probe: {
|
|
72
|
+
zone: dataArg.zone,
|
|
73
|
+
verdict: 'undeterminable' as const,
|
|
74
|
+
observedNameservers: [],
|
|
75
|
+
expectedNameservers: [],
|
|
76
|
+
detail: 'DnsAuthorityManager is not initialized',
|
|
77
|
+
},
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
return { probe: await manager.probeDelegation(dataArg.zone) };
|
|
81
|
+
},
|
|
82
|
+
),
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
// Claim authority over a zone, against a delegation proof
|
|
86
|
+
this.typedrouter.addTypedHandler(
|
|
87
|
+
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_VerifyDnsAuthorityZone>(
|
|
88
|
+
'verifyDnsAuthorityZone',
|
|
89
|
+
async (dataArg) => {
|
|
90
|
+
const userId = await this.requireAuth(dataArg, 'dns-authority:write');
|
|
91
|
+
const manager = this.opsServerRef.dcRouterRef.dnsAuthorityManager;
|
|
92
|
+
if (!manager) {
|
|
93
|
+
return { success: false, message: 'DnsAuthorityManager is not initialized' };
|
|
94
|
+
}
|
|
95
|
+
const result = await manager.verifyZone(dataArg.zone, userId);
|
|
96
|
+
if (result.success) {
|
|
97
|
+
this.opsServerRef.invalidateRealtime?.(['dns', 'routes'], {
|
|
98
|
+
reason: 'dns-authority-verified',
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
return result;
|
|
102
|
+
},
|
|
103
|
+
),
|
|
104
|
+
);
|
|
105
|
+
|
|
106
|
+
// Drop a verified zone. Every zone is revocable — there is no bootstrap floor.
|
|
107
|
+
this.typedrouter.addTypedHandler(
|
|
108
|
+
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_RevokeDnsAuthorityZone>(
|
|
109
|
+
'revokeDnsAuthorityZone',
|
|
110
|
+
async (dataArg) => {
|
|
111
|
+
const userId = await this.requireAuth(dataArg, 'dns-authority:write');
|
|
112
|
+
const manager = this.opsServerRef.dcRouterRef.dnsAuthorityManager;
|
|
113
|
+
if (!manager) {
|
|
114
|
+
return { success: false, message: 'DnsAuthorityManager is not initialized' };
|
|
115
|
+
}
|
|
116
|
+
const result = await manager.revokeZone(dataArg.zone, userId);
|
|
117
|
+
if (result.success) {
|
|
118
|
+
this.opsServerRef.invalidateRealtime?.(['dns', 'routes'], {
|
|
119
|
+
reason: 'dns-authority-revoked',
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
return result;
|
|
123
|
+
},
|
|
124
|
+
),
|
|
125
|
+
);
|
|
126
|
+
|
|
127
|
+
// Compare declared authority against observed delegation, both directions
|
|
128
|
+
this.typedrouter.addTypedHandler(
|
|
129
|
+
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetDnsAuthorityDrift>(
|
|
130
|
+
'getDnsAuthorityDrift',
|
|
131
|
+
async (dataArg) => {
|
|
132
|
+
await this.requireAuth(dataArg, 'dns-authority:read');
|
|
133
|
+
const manager = this.opsServerRef.dcRouterRef.dnsAuthorityManager;
|
|
134
|
+
if (!manager) {
|
|
135
|
+
return { drift: [] };
|
|
136
|
+
}
|
|
137
|
+
return { drift: await manager.auditDelegationDrift() };
|
|
138
|
+
},
|
|
139
|
+
),
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
@@ -138,6 +138,12 @@ export class DnsProviderHandler {
|
|
|
138
138
|
if (!dnsManager) return { success: false, message: 'DnsManager not initialized' };
|
|
139
139
|
const result = await dnsManager.deleteProvider(dataArg.id, dataArg.force ?? false);
|
|
140
140
|
if (result.success) {
|
|
141
|
+
// Removing a provider removes the credentialed zone listing that
|
|
142
|
+
// proved ownership for its domains, so the ownership-gated private
|
|
143
|
+
// route DNS overlay must be re-derived rather than left serving.
|
|
144
|
+
await this.opsServerRef.dcRouterRef.resyncPrivateRouteDnsOverlay(
|
|
145
|
+
'DNS provider deletion',
|
|
146
|
+
);
|
|
141
147
|
this.opsServerRef.invalidateRealtime?.(['dns', 'emailDomains'], {
|
|
142
148
|
changedIds: [dataArg.id],
|
|
143
149
|
reason: 'dns-provider-deleted',
|
|
@@ -138,6 +138,13 @@ export class DomainHandler {
|
|
|
138
138
|
if (!dnsManager) return { success: false, message: 'DnsManager not initialized' };
|
|
139
139
|
const ok = await dnsManager.deleteDomain(dataArg.id);
|
|
140
140
|
if (ok) {
|
|
141
|
+
// The private-route DNS overlay is derived from routes but gated on
|
|
142
|
+
// domain ownership, so deleting a domain invalidates it even though no
|
|
143
|
+
// route changed. Without this it keeps answering authoritatively for a
|
|
144
|
+
// zone we no longer manage until an unrelated route apply.
|
|
145
|
+
await this.opsServerRef.dcRouterRef.resyncPrivateRouteDnsOverlay(
|
|
146
|
+
'domain deletion',
|
|
147
|
+
);
|
|
141
148
|
this.opsServerRef.invalidateRealtime?.(['dns', 'emailDomains'], {
|
|
142
149
|
changedIds: [dataArg.id],
|
|
143
150
|
reason: 'domain-deleted',
|
|
@@ -183,6 +190,11 @@ export class DomainHandler {
|
|
|
183
190
|
deleteExistingProviderRecords: dataArg.deleteExistingProviderRecords,
|
|
184
191
|
});
|
|
185
192
|
if (result.success) {
|
|
193
|
+
// Migrating away from a provider can drop the proof that made the
|
|
194
|
+
// overlay's hostnames servable, so re-derive it here too.
|
|
195
|
+
await this.opsServerRef.dcRouterRef.resyncPrivateRouteDnsOverlay(
|
|
196
|
+
'domain migration',
|
|
197
|
+
);
|
|
186
198
|
this.opsServerRef.invalidateRealtime?.(['dns', 'emailDomains'], {
|
|
187
199
|
changedIds: [dataArg.id],
|
|
188
200
|
reason: 'domain-migrated',
|
|
@@ -604,7 +604,7 @@ export class GatewayClientHandler {
|
|
|
604
604
|
enabled: Boolean(dcRouter.remoteIngressManager?.getHubSettings().enabled),
|
|
605
605
|
},
|
|
606
606
|
dns: {
|
|
607
|
-
authoritative: Boolean(dcRouter.
|
|
607
|
+
authoritative: Boolean(dcRouter.dnsAuthorityManager?.getEffectiveZoneNames().length),
|
|
608
608
|
providerManaged: Boolean(dcRouter.dnsManager),
|
|
609
609
|
},
|
|
610
610
|
http3: {
|
|
@@ -17,6 +17,7 @@ export * from './network-target.handler.js';
|
|
|
17
17
|
export * from './users.handler.js';
|
|
18
18
|
export * from './dns-provider.handler.js';
|
|
19
19
|
export * from './domain.handler.js';
|
|
20
|
+
export * from './dns-authority.handler.js';
|
|
20
21
|
export * from './dns-record.handler.js';
|
|
21
22
|
export * from './acme-config.handler.js';
|
|
22
23
|
export * from './email-domain.handler.js';
|
package/ts/readme.md
CHANGED
|
@@ -58,7 +58,7 @@ await router.start();
|
|
|
58
58
|
- SmartProxy for HTTP/HTTPS/TCP routes
|
|
59
59
|
- `UnifiedEmailServer` for SMTP ingress, delivery, managed app address bindings, and outbound SMTP submission when `emailConfig` is present
|
|
60
60
|
- DB-backed managers for routes, API tokens, target profiles, domains, records, ACME config, and email domains when the DB is enabled
|
|
61
|
-
- embedded authoritative DNS and DoH route generation from `dnsNsDomains`
|
|
61
|
+
- embedded authoritative DNS and DoH route generation from `dnsNsDomains` plus the delegation-verified zones held in the database
|
|
62
62
|
- VPN, RADIUS, and remote ingress services when their config blocks are enabled
|
|
63
63
|
- OpsServer and the dashboard, which start on every boot
|
|
64
64
|
- an admin-JWT authenticated read-only MCP endpoint at `/mcp` for safe route, DNS, email, RemoteIngress, and VPN summaries
|