@serve.zone/dcrouter 17.10.2 → 18.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/deno.json +1 -1
  2. package/dist_serve/bundle.js +360 -360
  3. package/dist_ts/00_commitinfo_data.js +2 -2
  4. package/dist_ts/acme/acme-failure-classification.d.ts +64 -0
  5. package/dist_ts/acme/acme-failure-classification.js +114 -0
  6. package/dist_ts/acme/classes.smartacme-lifecycle.d.ts +42 -0
  7. package/dist_ts/acme/classes.smartacme-lifecycle.js +75 -4
  8. package/dist_ts/acme/index.d.ts +1 -0
  9. package/dist_ts/acme/index.js +2 -1
  10. package/dist_ts/classes.dcrouter.d.ts +33 -9
  11. package/dist_ts/classes.dcrouter.js +145 -11
  12. package/dist_ts/config/classes.route-config-manager.d.ts +56 -0
  13. package/dist_ts/config/classes.route-config-manager.js +161 -1
  14. package/dist_ts/db/documents/classes.dns-authority.doc.d.ts +30 -0
  15. package/dist_ts/db/documents/classes.dns-authority.doc.js +108 -0
  16. package/dist_ts/db/documents/index.d.ts +1 -0
  17. package/dist_ts/db/documents/index.js +2 -1
  18. package/dist_ts/dns/classes.dns-server-runtime.d.ts +109 -1
  19. package/dist_ts/dns/classes.dns-server-runtime.js +212 -34
  20. package/dist_ts/dns/domain-ownership.d.ts +111 -0
  21. package/dist_ts/dns/domain-ownership.js +152 -0
  22. package/dist_ts/dns/index.d.ts +2 -0
  23. package/dist_ts/dns/index.js +3 -1
  24. package/dist_ts/dns/manager.dns-authority.d.ts +143 -0
  25. package/dist_ts/dns/manager.dns-authority.js +481 -0
  26. package/dist_ts/dns/manager.dns.d.ts +121 -12
  27. package/dist_ts/dns/manager.dns.js +298 -28
  28. package/dist_ts/email/classes.accepted-email-spool.d.ts +15 -0
  29. package/dist_ts/email/classes.accepted-email-spool.js +102 -23
  30. package/dist_ts/email/classes.smartmta-blob-storage-manager.js +12 -3
  31. package/dist_ts/errors/error.codes.d.ts +4 -0
  32. package/dist_ts/errors/error.codes.js +5 -1
  33. package/dist_ts/opsserver/classes.opsserver.d.ts +1 -0
  34. package/dist_ts/opsserver/classes.opsserver.js +3 -1
  35. package/dist_ts/opsserver/handlers/acme-config.handler.js +6 -1
  36. package/dist_ts/opsserver/handlers/certificate.handler.d.ts +16 -0
  37. package/dist_ts/opsserver/handlers/certificate.handler.js +96 -10
  38. package/dist_ts/opsserver/handlers/config.handler.js +4 -2
  39. package/dist_ts/opsserver/handlers/dns-authority.handler.d.ts +20 -0
  40. package/dist_ts/opsserver/handlers/dns-authority.handler.js +108 -0
  41. package/dist_ts/opsserver/handlers/dns-provider.handler.js +5 -1
  42. package/dist_ts/opsserver/handlers/domain.handler.js +9 -1
  43. package/dist_ts/opsserver/handlers/gatewayclient.handler.js +2 -2
  44. package/dist_ts/opsserver/handlers/index.d.ts +1 -0
  45. package/dist_ts/opsserver/handlers/index.js +2 -1
  46. package/dist_ts_interfaces/data/dns-authority.d.ts +98 -0
  47. package/dist_ts_interfaces/data/dns-authority.js +26 -0
  48. package/dist_ts_interfaces/data/index.d.ts +1 -0
  49. package/dist_ts_interfaces/data/index.js +2 -1
  50. package/dist_ts_interfaces/data/route-management.d.ts +9 -2
  51. package/dist_ts_interfaces/data/route-management.js +3 -1
  52. package/dist_ts_interfaces/requests/certificate.d.ts +23 -0
  53. package/dist_ts_interfaces/requests/certificate.js +1 -1
  54. package/dist_ts_interfaces/requests/dns-authority.d.ts +80 -0
  55. package/dist_ts_interfaces/requests/dns-authority.js +3 -0
  56. package/dist_ts_interfaces/requests/index.d.ts +1 -0
  57. package/dist_ts_interfaces/requests/index.js +2 -1
  58. package/dist_ts_oci_container/index.js +9 -4
  59. package/dist_ts_web/00_commitinfo_data.js +2 -2
  60. package/package.json +1 -1
  61. package/readme.hints.md +412 -0
  62. package/readme.md +48 -6
  63. package/ts/00_commitinfo_data.ts +1 -1
  64. package/ts/acme/acme-failure-classification.ts +201 -0
  65. package/ts/acme/classes.smartacme-lifecycle.ts +104 -3
  66. package/ts/acme/index.ts +1 -0
  67. package/ts/classes.dcrouter.ts +193 -23
  68. package/ts/config/classes.route-config-manager.ts +197 -0
  69. package/ts/db/documents/classes.dns-authority.doc.ts +49 -0
  70. package/ts/db/documents/index.ts +1 -0
  71. package/ts/dns/classes.dns-server-runtime.ts +257 -38
  72. package/ts/dns/domain-ownership.ts +272 -0
  73. package/ts/dns/index.ts +2 -0
  74. package/ts/dns/manager.dns-authority.ts +558 -0
  75. package/ts/dns/manager.dns.ts +373 -27
  76. package/ts/email/classes.accepted-email-spool.ts +98 -21
  77. package/ts/email/classes.smartmta-blob-storage-manager.ts +10 -2
  78. package/ts/errors/error.codes.ts +4 -0
  79. package/ts/opsserver/classes.opsserver.ts +2 -0
  80. package/ts/opsserver/handlers/acme-config.handler.ts +7 -0
  81. package/ts/opsserver/handlers/certificate.handler.ts +103 -8
  82. package/ts/opsserver/handlers/config.handler.ts +3 -1
  83. package/ts/opsserver/handlers/dns-authority.handler.ts +142 -0
  84. package/ts/opsserver/handlers/dns-provider.handler.ts +6 -0
  85. package/ts/opsserver/handlers/domain.handler.ts +12 -0
  86. package/ts/opsserver/handlers/gatewayclient.handler.ts +1 -1
  87. package/ts/opsserver/handlers/index.ts +1 -0
  88. package/ts/readme.md +1 -1
  89. package/ts_web/00_commitinfo_data.ts +1 -1
@@ -77,6 +77,19 @@ const ACCEPTED_EMAIL_STOP_DRAIN_TIMEOUT_MS = 30_000;
77
77
  /** Retention for catch-all stored inbound mail (30 days). */
78
78
  const INBOUND_STORE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
79
79
 
80
+ /**
81
+ * Permanent per-email storage failure: the raw RFC822 payload of an accepted
82
+ * email is unrecoverable, so delivery can never succeed. The spool marks such
83
+ * rows failed instead of retrying them, so one lost blob cannot wedge the
84
+ * whole queue.
85
+ */
86
+ export class AcceptedEmailRawMessageMissingError extends Error {
87
+ constructor(messageArg: string) {
88
+ super(messageArg);
89
+ this.name = 'AcceptedEmailRawMessageMissingError';
90
+ }
91
+ }
92
+
80
93
  export type TSmartMtaQueueItemLike = {
81
94
  id?: string;
82
95
  processingResult?: {
@@ -690,32 +703,55 @@ export class AcceptedEmailSpool {
690
703
  for (const item of (emailServerArg as TSmartMtaQueueReader).getQueueItems?.() || []) {
691
704
  const cachedEmailId = this.getCachedEmailIdFromQueueItem(item);
692
705
  if (!cachedEmailId || !item.smtpTransactions?.length) continue;
693
- await this.runCachedEmailUpdate(cachedEmailId, async () => {
694
- const cachedEmail = await CachedEmail.findById(cachedEmailId);
695
- if (!cachedEmail) return;
696
- for (const transaction of item.smtpTransactions || []) {
697
- this.appendSmtpTransaction(cachedEmail, transaction);
698
- }
699
- await cachedEmail.save();
700
- });
706
+ try {
707
+ await this.runCachedEmailUpdate(cachedEmailId, async () => {
708
+ const cachedEmail = await CachedEmail.findById(cachedEmailId);
709
+ if (!cachedEmail) return;
710
+ for (const transaction of item.smtpTransactions || []) {
711
+ this.appendSmtpTransaction(cachedEmail, transaction);
712
+ }
713
+ await cachedEmail.save();
714
+ });
715
+ } catch (error: unknown) {
716
+ // History reconciliation runs during email-server startup; one row that
717
+ // cannot be persisted must not abort reconciliation or startup.
718
+ logger.log('warn', `Unable to reconcile SMTP transaction history for accepted email ${cachedEmailId}: ${(error as Error).message}`);
719
+ }
701
720
  }
702
721
  }
703
722
 
704
723
  /** Requeue emails left in 'queued' state by a previous process as pending. */
705
724
  public async recoverQueuedEmails(): Promise<void> {
725
+ // Rows whose recovery save failed stay 'queued' and are re-served by
726
+ // findQueuedForRecovery; track them so they are neither retried in a loop
727
+ // within this run nor able to wedge recovery when they fill a whole batch.
728
+ const unrecoverableIds = new Set<string>();
706
729
  while (true) {
707
730
  const queuedEmails = await CachedEmail.findQueuedForRecovery(ACCEPTED_EMAIL_SPOOL_BATCH_SIZE);
708
- if (queuedEmails.length === 0) {
731
+ const recoverableEmails = queuedEmails.filter((queuedEmail) => !unrecoverableIds.has(queuedEmail.id));
732
+ if (recoverableEmails.length === 0) {
709
733
  return;
710
734
  }
711
- for (const queuedEmail of queuedEmails) {
735
+ for (const queuedEmail of recoverableEmails) {
712
736
  if (this.isCachedEmailTerminal(queuedEmail)) {
713
737
  continue;
714
738
  }
715
- queuedEmail.status = 'pending';
716
- queuedEmail.nextAttempt = new Date();
717
- await queuedEmail.save();
718
- await this.notifyEmailQueuePersisted(queuedEmail, 'email-queue-recovered');
739
+ const previousStatus = queuedEmail.status;
740
+ const previousNextAttempt = queuedEmail.nextAttempt;
741
+ try {
742
+ queuedEmail.status = 'pending';
743
+ queuedEmail.nextAttempt = new Date();
744
+ await queuedEmail.save();
745
+ await this.notifyEmailQueuePersisted(queuedEmail, 'email-queue-recovered');
746
+ } catch (error: unknown) {
747
+ // Recovery runs during email-server startup; a row that cannot be
748
+ // persisted stays 'queued' for the next startup and must not abort
749
+ // recovery of the remaining rows.
750
+ queuedEmail.status = previousStatus;
751
+ queuedEmail.nextAttempt = previousNextAttempt;
752
+ unrecoverableIds.add(queuedEmail.id);
753
+ logger.log('warn', `Unable to recover queued accepted email ${queuedEmail.id}: ${(error as Error).message}`);
754
+ }
719
755
  }
720
756
  if (queuedEmails.length < ACCEPTED_EMAIL_SPOOL_BATCH_SIZE) {
721
757
  return;
@@ -735,18 +771,54 @@ export class AcceptedEmailSpool {
735
771
  if (this.stopping || this.dcRouterRef.emailServer !== emailServer) {
736
772
  break;
737
773
  }
738
- if (await this.postponeLiveSmartMtaOwnedEmail(cachedEmail, emailServer)) {
739
- continue;
774
+ try {
775
+ if (await this.postponeLiveSmartMtaOwnedEmail(cachedEmail, emailServer)) {
776
+ continue;
777
+ }
778
+ const session = this.buildCachedEmailSession(cachedEmail);
779
+ const rawMessage = await this.readRawMessage(cachedEmail);
780
+ await this.processAcceptedCachedEmail(cachedEmail, rawMessage, session, emailServer);
781
+ } catch (error: unknown) {
782
+ // One poisoned row must never stall the rest of the batch: record the
783
+ // failure on that row alone and keep processing the remaining emails.
784
+ await this.handleSpoolItemFailure(cachedEmail, error);
740
785
  }
741
- const session = this.buildCachedEmailSession(cachedEmail);
742
- const rawMessage = await this.readRawMessage(cachedEmail);
743
- await this.processAcceptedCachedEmail(cachedEmail, rawMessage, session, emailServer);
744
786
  }
745
787
  } finally {
746
788
  this.processing = false;
747
789
  }
748
790
  }
749
791
 
792
+ /**
793
+ * Record a per-email spool failure without aborting the batch: permanent
794
+ * raw-message losses are marked failed, everything else is deferred through
795
+ * the regular retry mechanism (which terminates at maxAttempts).
796
+ */
797
+ private async handleSpoolItemFailure(cachedEmail: CachedEmail, errorArg: unknown): Promise<void> {
798
+ const message = (errorArg as Error).message;
799
+ try {
800
+ const currentCachedEmail = await CachedEmail.findById(cachedEmail.id) || cachedEmail;
801
+ if (this.isCachedEmailTerminal(currentCachedEmail)) {
802
+ return;
803
+ }
804
+ if (errorArg instanceof AcceptedEmailRawMessageMissingError) {
805
+ currentCachedEmail.markFailed(message);
806
+ await currentCachedEmail.save();
807
+ await this.notifyEmailQueuePersisted(currentCachedEmail, 'email-raw-message-missing');
808
+ logger.log('error', `Accepted email ${currentCachedEmail.id} failed permanently: ${message}`);
809
+ return;
810
+ }
811
+ currentCachedEmail.scheduleRetry(ACCEPTED_EMAIL_RETRY_DELAY_MS);
812
+ currentCachedEmail.lastError = message;
813
+ await currentCachedEmail.save();
814
+ await this.notifyEmailQueuePersisted(currentCachedEmail, 'email-spool-deferred');
815
+ logger.log('warn', `Accepted email ${currentCachedEmail.id} deferred after spool processing failure: ${message}`);
816
+ } catch (persistError: unknown) {
817
+ // Even failing to persist the failure state must not abort the batch.
818
+ logger.log('error', `Unable to persist spool failure state for accepted email ${cachedEmail.id}: ${(persistError as Error).message} (original error: ${message})`);
819
+ }
820
+ }
821
+
750
822
  private async processAcceptedCachedEmail(
751
823
  cachedEmail: CachedEmail,
752
824
  emailData: Email | plugins.buffer.Buffer,
@@ -1010,11 +1082,16 @@ export class AcceptedEmailSpool {
1010
1082
  }
1011
1083
  const rawMessage = await blobStorage.get(cachedEmailArg.rawContentObjectKey);
1012
1084
  if (!rawMessage) {
1013
- throw new Error(`Raw RFC822 object is missing for accepted email ${cachedEmailArg.id}`);
1085
+ throw new AcceptedEmailRawMessageMissingError(`Raw RFC822 object is missing for accepted email ${cachedEmailArg.id}`);
1014
1086
  }
1015
1087
  return plugins.buffer.Buffer.from(rawMessage);
1016
1088
  }
1017
- throw new Error(`Accepted email ${cachedEmailArg.id} has no raw RFC822 payload`);
1089
+ if (cachedEmailArg.rawContent) {
1090
+ // Legacy rows persisted before SmartBucket-backed storage keep their
1091
+ // payload inline; serve it rather than declaring the payload lost.
1092
+ return plugins.buffer.Buffer.from(cachedEmailArg.rawContent, 'utf8');
1093
+ }
1094
+ throw new AcceptedEmailRawMessageMissingError(`Accepted email ${cachedEmailArg.id} has no raw RFC822 payload`);
1018
1095
  }
1019
1096
 
1020
1097
  public async deleteRawMessage(cachedEmailArg: CachedEmail): Promise<void> {
@@ -1,4 +1,5 @@
1
1
  import * as plugins from '../plugins.js';
2
+ import { logger } from '../logger.js';
2
3
  import type { IBlobStorageManager } from '@push.rocks/smartmta';
3
4
 
4
5
  const SMARTMTA_BLOB_NAMESPACES = [
@@ -119,11 +120,18 @@ export class SmartMtaBlobStorageManager implements IBlobStorageManager {
119
120
  const objectPrefix = this.toObjectKey(prefix);
120
121
  const keys: string[] = [];
121
122
  for await (const objectKey of this.bucketRef.listAllObjects(objectPrefix)) {
123
+ // One stray or malformed object under the prefix must not abort the
124
+ // whole listing (SmartMTA queue recovery lists at startup); skip it.
122
125
  if (!objectKey.startsWith(this.objectPrefix)) {
123
- throw new Error(`SmartBucket returned an object outside the configured prefix: ${objectKey}`);
126
+ logger.log('warn', `Skipping SmartBucket object outside the configured prefix during SmartMTA blob listing: ${objectKey}`);
127
+ continue;
124
128
  }
125
129
  const key = `/${objectKey.slice(this.objectPrefix.length)}`;
126
- keys.push(normalizeBlobKey(key));
130
+ try {
131
+ keys.push(normalizeBlobKey(key));
132
+ } catch (error: unknown) {
133
+ logger.log('warn', `Skipping unnormalizable SmartBucket object key during SmartMTA blob listing: ${objectKey} (${(error as Error).message})`);
134
+ }
127
135
  }
128
136
  return keys.sort();
129
137
  }
@@ -80,6 +80,10 @@ export const DCR_ROUTING_ERROR = 'DCR_ROUTING_ERROR';
80
80
  export const DCR_CONFIGURATION_ERROR = 'DCR_CONFIGURATION_ERROR';
81
81
  export const DCR_PROXY_ERROR = 'DCR_PROXY_ERROR';
82
82
  export const DCR_DOMAIN_ERROR = 'DCR_DOMAIN_ERROR';
83
+ /** A certificate requirement or authoritative DNS zone was requested for a domain we cannot prove we own. */
84
+ export const DCR_DOMAIN_OWNERSHIP_UNVERIFIED = 'DCR_DOMAIN_OWNERSHIP_UNVERIFIED';
85
+ /** ACME failed for a configuration cause that no amount of retrying can resolve. */
86
+ export const DCR_ACME_PERMANENT_FAILURE = 'DCR_ACME_PERMANENT_FAILURE';
83
87
 
84
88
  // SMS service errors (SMS_*)
85
89
  export const SMS_SERVICE_ERROR = 'SMS_SERVICE_ERROR';
@@ -37,6 +37,7 @@ export class OpsServer {
37
37
  private usersHandler!: handlers.UsersHandler;
38
38
  private dnsProviderHandler!: handlers.DnsProviderHandler;
39
39
  private domainHandler!: handlers.DomainHandler;
40
+ private dnsAuthorityHandler!: handlers.DnsAuthorityHandler;
40
41
  private dnsRecordHandler!: handlers.DnsRecordHandler;
41
42
  private acmeConfigHandler!: handlers.AcmeConfigHandler;
42
43
  private emailDomainHandler!: handlers.EmailDomainHandler;
@@ -104,6 +105,7 @@ export class OpsServer {
104
105
  this.usersHandler = new handlers.UsersHandler(this);
105
106
  this.dnsProviderHandler = new handlers.DnsProviderHandler(this);
106
107
  this.domainHandler = new handlers.DomainHandler(this);
108
+ this.dnsAuthorityHandler = new handlers.DnsAuthorityHandler(this);
107
109
  this.dnsRecordHandler = new handlers.DnsRecordHandler(this);
108
110
  this.acmeConfigHandler = new handlers.AcmeConfigHandler(this);
109
111
  this.emailDomainHandler = new handlers.EmailDomainHandler(this);
@@ -66,6 +66,13 @@ export class AcmeConfigHandler {
66
66
  },
67
67
  userId,
68
68
  );
69
+ // The SmartAcme startup budget does not re-arm on a timer, so a
70
+ // corrected ACME configuration must re-arm it explicitly. Otherwise a
71
+ // permanently-failed or exhausted provider stays down until an
72
+ // unrelated SmartProxy rebuild or a restart (30–60 s of outage).
73
+ this.opsServerRef.dcRouterRef.smartAcmeLifecycle?.rearm(
74
+ 'ACME configuration updated',
75
+ );
69
76
  return { success: true, config: updated };
70
77
  } catch (err: unknown) {
71
78
  return { success: false, message: (err as Error).message };
@@ -266,22 +266,35 @@ export class CertificateHandler {
266
266
  }
267
267
  }
268
268
 
269
- // Try SmartProxy certificate status if no event data
270
- if (status === 'unknown' && info.routeNames.length > 0) {
269
+ // What the running proxy actually serves. Always queried, not only as a
270
+ // fallback: the served state is an independent fact from the stored one and
271
+ // the two must be comparable.
272
+ let served: interfaces.requests.ICertificateServedState | undefined;
273
+ if (info.routeNames.length > 0) {
271
274
  try {
272
275
  const rustStatus = await smartProxy.getCertificateStatus(info.routeNames[0]);
273
276
  if (rustStatus) {
274
- if (rustStatus.expiresAt > 0) {
275
- expiryDate = new Date(rustStatus.expiresAt).toISOString();
276
- }
277
- if (rustStatus.source) issuer = rustStatus.source;
278
- status = rustStatus.isValid ? 'valid' : 'expired';
277
+ served = {
278
+ expiryDate: rustStatus.expiresAt > 0
279
+ ? new Date(rustStatus.expiresAt).toISOString()
280
+ : undefined,
281
+ isValid: rustStatus.isValid,
282
+ source: rustStatus.source || undefined,
283
+ };
279
284
  }
280
285
  } catch {
281
- // Rust bridge may not support this command yet — ignore
286
+ // Rust bridge may not support this command yet — leave served unknown.
282
287
  }
283
288
  }
284
289
 
290
+ // With no event data, the served state is also the best available view of
291
+ // the issued one.
292
+ if (status === 'unknown' && served) {
293
+ if (served.expiryDate) expiryDate = served.expiryDate;
294
+ if (served.source) issuer = served.source;
295
+ status = served.isValid ? 'valid' : 'expired';
296
+ }
297
+
285
298
  // Check persisted cert data from smartdata document classes
286
299
  if (status === 'unknown') {
287
300
  const cleanDomain = domain.replace(/^\*\.?/, '');
@@ -352,6 +365,23 @@ export class CertificateHandler {
352
365
  error = error || backoffInfo.lastError;
353
366
  }
354
367
 
368
+ // Issued is not served. A certificate valid in storage while the engine
369
+ // still presents an invalid or expired one is a failure, however healthy
370
+ // the issuance path looks — this is exactly what hid a 21-day outage.
371
+ const servedMismatch = Boolean(
372
+ served
373
+ && !served.isValid
374
+ && (status === 'valid' || status === 'expiring'),
375
+ );
376
+ if (servedMismatch) {
377
+ status = 'failed';
378
+ error = error
379
+ || `A valid certificate is stored for '${domain}'`
380
+ + `${expiryDate ? ` (expires ${expiryDate})` : ''}, but the running proxy is serving an invalid one`
381
+ + `${served?.expiryDate ? ` (expired ${served.expiryDate})` : ''}. `
382
+ + 'The issued certificate has not reached the engine.';
383
+ }
384
+
355
385
  certificates.push({
356
386
  domain,
357
387
  routeNames: info.routeNames,
@@ -363,6 +393,8 @@ export class CertificateHandler {
363
393
  issuedAt,
364
394
  error,
365
395
  canReprovision: info.canReprovision,
396
+ served,
397
+ ...(servedMismatch ? { servedMismatch: true } : {}),
366
398
  backoffInfo,
367
399
  });
368
400
  }
@@ -419,6 +451,21 @@ export class CertificateHandler {
419
451
  return { success: false, message: `No routes found for domain '${domain}'` };
420
452
  }
421
453
 
454
+ // Refuse before triggering an order that cannot succeed. forceRenew is an
455
+ // operator override for backoff, not for missing ownership: without a zone we
456
+ // can prove is ours there is nowhere to place the DNS-01 challenge record.
457
+ if (dcRouter.dnsManager) {
458
+ const ownership = await dcRouter.dnsManager.resolveDomainOwnership(domain);
459
+ if (!ownership.verified) {
460
+ return {
461
+ success: false,
462
+ message:
463
+ `Cannot provision a certificate for '${domain}': domain ownership is unverified `
464
+ + `(${ownership.reason}) — ${ownership.detail}`,
465
+ };
466
+ }
467
+ }
468
+
422
469
  // Respect existing backoff unless explicitly overridden. Automated
423
470
  // callers (gateway clients retrying cert fetches) used to reset the
424
471
  // backoff and trigger a route re-apply on every request, producing a
@@ -488,12 +535,60 @@ export class CertificateHandler {
488
535
  // Fallback when DB is disabled and there is no RouteConfigManager
489
536
  await smartProxy.updateRoutes(smartProxy.routeManager.getRoutes());
490
537
  }
538
+ const propagation = await this.describeCertPropagationGap(domain, routeNames);
539
+ if (propagation) {
540
+ return { success: false, message: propagation };
541
+ }
491
542
  return { success: true, message: forceRenew ? `Certificate force-renewed for domain '${domain}'` : `Certificate reprovisioning triggered for domain '${domain}'` };
492
543
  } catch (err: unknown) {
493
544
  return { success: false, message: (err as Error).message || `Failed to reprovision certificate for ${domain}` };
494
545
  }
495
546
  }
496
547
 
548
+ /**
549
+ * After triggering the apply pipeline, check whether the certificate the proxy
550
+ * is actually serving is still expired. If it is, the reprovision did not
551
+ * propagate and reporting success would be a lie.
552
+ *
553
+ * The known cause is SmartProxy's own per-domain provisioning cooldown
554
+ * (`certProvisionFailureCooldownMs`, 30 min default). It is private in-memory
555
+ * state with no invalidation path: a provisioning sweep skips a cooling-down
556
+ * domain, and because the sweep returns before its summary log when every
557
+ * domain is skipped, the skip is not logged either. So a valid certificate can
558
+ * sit in storage while the proxy keeps serving the expired one, silently.
559
+ *
560
+ * Returns undefined when propagation looks fine or cannot be determined —
561
+ * `getCertificateStatus` goes through the Rust bridge and may be unavailable.
562
+ */
563
+ private async describeCertPropagationGap(
564
+ domain: string,
565
+ routeNames: string[],
566
+ ): Promise<string | undefined> {
567
+ const smartProxy = this.opsServerRef.dcRouterRef.smartProxy;
568
+ if (!smartProxy || routeNames.length === 0) {
569
+ return undefined;
570
+ }
571
+ let status: Awaited<ReturnType<typeof smartProxy.getCertificateStatus>>;
572
+ try {
573
+ status = await smartProxy.getCertificateStatus(routeNames[0]);
574
+ } catch {
575
+ // Bridge command unsupported — cannot determine, so do not claim a gap.
576
+ return undefined;
577
+ }
578
+ if (!status || status.isValid || !status.expiresAt || status.expiresAt <= 0) {
579
+ return undefined;
580
+ }
581
+ const servedUntil = new Date(status.expiresAt).toISOString();
582
+ const message =
583
+ `Certificate for '${domain}' was issued and stored, but route '${routeNames[0]}' is still serving a `
584
+ + `certificate that expired at ${servedUntil}. The new certificate did not reach the running proxy: `
585
+ + 'SmartProxy keeps a private per-domain provisioning cooldown that a sweep cannot clear while it is '
586
+ + 'active, and it exposes no per-domain invalidation or hot-load API. Toggling this single route off and '
587
+ + 'back on is the only supported way to force a fresh attempt without restarting dcrouter.';
588
+ logger.log('error', message, { domain, routeName: routeNames[0] });
589
+ return message;
590
+ }
591
+
497
592
  /**
498
593
  * After a force-renew, walk every route in the smartproxy that resolves to
499
594
  * the same cert identity as `forcedDomain` and write the freshly-issued cert
@@ -137,7 +137,9 @@ export class ConfigHandler {
137
137
  enabled: !!dcRouter.dnsServer,
138
138
  port: 53,
139
139
  nsDomains: opts.dnsNsDomains || [],
140
- scopes: opts.dnsScopes || [],
140
+ // The delegation-verified authority set. Deployment configuration no
141
+ // longer contributes: this is database state, and it is the only state.
142
+ scopes: dcRouter.dnsAuthorityManager?.getEffectiveZoneNames() || [],
141
143
  recordCount: dnsRecords.length,
142
144
  records: dnsRecords,
143
145
  dnsChallenge: dnsChallengeEnabled,
@@ -0,0 +1,142 @@
1
+ import * as plugins from '../../plugins.js';
2
+ import type { OpsServer } from '../classes.opsserver.js';
3
+ import * as interfaces from '../../../ts_interfaces/index.js';
4
+ import { requireOpsAuth } from '../helpers/auth.js';
5
+
6
+ /**
7
+ * DNS authority handlers.
8
+ *
9
+ * Adding a zone here is an authority claim — after it succeeds dcrouter answers
10
+ * for that zone with the `aa` flag on a public port and will request
11
+ * certificates for it. So it carries the same posture as the other DNS write
12
+ * surfaces (admin identity or a `:write`-scoped token) *and* the claim itself is
13
+ * gated on a delegation proof the caller cannot forge. Authorization decides who
14
+ * may ask; the probe decides whether the answer is yes.
15
+ */
16
+ export class DnsAuthorityHandler {
17
+ public typedrouter = new plugins.typedrequest.TypedRouter();
18
+
19
+ constructor(private opsServerRef: OpsServer) {
20
+ this.opsServerRef.typedrouter.addTypedRouter(this.typedrouter);
21
+ this.registerHandlers();
22
+ }
23
+
24
+ private async requireAuth(
25
+ request: { identity?: interfaces.data.IIdentity; apiToken?: string },
26
+ requiredScope?: interfaces.data.TApiTokenScope,
27
+ ): Promise<string> {
28
+ const auth = await requireOpsAuth(this.opsServerRef, request, {
29
+ scope: requiredScope,
30
+ requireAdminIdentity: requiredScope?.endsWith(':write'),
31
+ });
32
+ return auth.userId;
33
+ }
34
+
35
+ private registerHandlers(): void {
36
+ // Read the authority set: delegation-verified zones, and nothing else.
37
+ this.typedrouter.addTypedHandler(
38
+ new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetDnsAuthority>(
39
+ 'getDnsAuthority',
40
+ async (dataArg) => {
41
+ await this.requireAuth(dataArg, 'dns-authority:read');
42
+ const manager = this.opsServerRef.dcRouterRef.dnsAuthorityManager;
43
+ if (!manager) {
44
+ // No manager means the set was never loaded. Reported as
45
+ // 'unavailable' rather than as an empty set, so a reader cannot
46
+ // mistake "we could not look" for "there is nothing".
47
+ return {
48
+ settings: {
49
+ zones: [],
50
+ state: 'unavailable' as const,
51
+ expectedNameservers: [],
52
+ updatedAt: 0,
53
+ updatedBy: '',
54
+ },
55
+ };
56
+ }
57
+ return { settings: manager.getSettings() };
58
+ },
59
+ ),
60
+ );
61
+
62
+ // Probe a zone's delegation without changing anything
63
+ this.typedrouter.addTypedHandler(
64
+ new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_ProbeDnsAuthorityZone>(
65
+ 'probeDnsAuthorityZone',
66
+ async (dataArg) => {
67
+ await this.requireAuth(dataArg, 'dns-authority:read');
68
+ const manager = this.opsServerRef.dcRouterRef.dnsAuthorityManager;
69
+ if (!manager) {
70
+ return {
71
+ probe: {
72
+ zone: dataArg.zone,
73
+ verdict: 'undeterminable' as const,
74
+ observedNameservers: [],
75
+ expectedNameservers: [],
76
+ detail: 'DnsAuthorityManager is not initialized',
77
+ },
78
+ };
79
+ }
80
+ return { probe: await manager.probeDelegation(dataArg.zone) };
81
+ },
82
+ ),
83
+ );
84
+
85
+ // Claim authority over a zone, against a delegation proof
86
+ this.typedrouter.addTypedHandler(
87
+ new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_VerifyDnsAuthorityZone>(
88
+ 'verifyDnsAuthorityZone',
89
+ async (dataArg) => {
90
+ const userId = await this.requireAuth(dataArg, 'dns-authority:write');
91
+ const manager = this.opsServerRef.dcRouterRef.dnsAuthorityManager;
92
+ if (!manager) {
93
+ return { success: false, message: 'DnsAuthorityManager is not initialized' };
94
+ }
95
+ const result = await manager.verifyZone(dataArg.zone, userId);
96
+ if (result.success) {
97
+ this.opsServerRef.invalidateRealtime?.(['dns', 'routes'], {
98
+ reason: 'dns-authority-verified',
99
+ });
100
+ }
101
+ return result;
102
+ },
103
+ ),
104
+ );
105
+
106
+ // Drop a verified zone. Every zone is revocable — there is no bootstrap floor.
107
+ this.typedrouter.addTypedHandler(
108
+ new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_RevokeDnsAuthorityZone>(
109
+ 'revokeDnsAuthorityZone',
110
+ async (dataArg) => {
111
+ const userId = await this.requireAuth(dataArg, 'dns-authority:write');
112
+ const manager = this.opsServerRef.dcRouterRef.dnsAuthorityManager;
113
+ if (!manager) {
114
+ return { success: false, message: 'DnsAuthorityManager is not initialized' };
115
+ }
116
+ const result = await manager.revokeZone(dataArg.zone, userId);
117
+ if (result.success) {
118
+ this.opsServerRef.invalidateRealtime?.(['dns', 'routes'], {
119
+ reason: 'dns-authority-revoked',
120
+ });
121
+ }
122
+ return result;
123
+ },
124
+ ),
125
+ );
126
+
127
+ // Compare declared authority against observed delegation, both directions
128
+ this.typedrouter.addTypedHandler(
129
+ new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetDnsAuthorityDrift>(
130
+ 'getDnsAuthorityDrift',
131
+ async (dataArg) => {
132
+ await this.requireAuth(dataArg, 'dns-authority:read');
133
+ const manager = this.opsServerRef.dcRouterRef.dnsAuthorityManager;
134
+ if (!manager) {
135
+ return { drift: [] };
136
+ }
137
+ return { drift: await manager.auditDelegationDrift() };
138
+ },
139
+ ),
140
+ );
141
+ }
142
+ }
@@ -138,6 +138,12 @@ export class DnsProviderHandler {
138
138
  if (!dnsManager) return { success: false, message: 'DnsManager not initialized' };
139
139
  const result = await dnsManager.deleteProvider(dataArg.id, dataArg.force ?? false);
140
140
  if (result.success) {
141
+ // Removing a provider removes the credentialed zone listing that
142
+ // proved ownership for its domains, so the ownership-gated private
143
+ // route DNS overlay must be re-derived rather than left serving.
144
+ await this.opsServerRef.dcRouterRef.resyncPrivateRouteDnsOverlay(
145
+ 'DNS provider deletion',
146
+ );
141
147
  this.opsServerRef.invalidateRealtime?.(['dns', 'emailDomains'], {
142
148
  changedIds: [dataArg.id],
143
149
  reason: 'dns-provider-deleted',
@@ -138,6 +138,13 @@ export class DomainHandler {
138
138
  if (!dnsManager) return { success: false, message: 'DnsManager not initialized' };
139
139
  const ok = await dnsManager.deleteDomain(dataArg.id);
140
140
  if (ok) {
141
+ // The private-route DNS overlay is derived from routes but gated on
142
+ // domain ownership, so deleting a domain invalidates it even though no
143
+ // route changed. Without this it keeps answering authoritatively for a
144
+ // zone we no longer manage until an unrelated route apply.
145
+ await this.opsServerRef.dcRouterRef.resyncPrivateRouteDnsOverlay(
146
+ 'domain deletion',
147
+ );
141
148
  this.opsServerRef.invalidateRealtime?.(['dns', 'emailDomains'], {
142
149
  changedIds: [dataArg.id],
143
150
  reason: 'domain-deleted',
@@ -183,6 +190,11 @@ export class DomainHandler {
183
190
  deleteExistingProviderRecords: dataArg.deleteExistingProviderRecords,
184
191
  });
185
192
  if (result.success) {
193
+ // Migrating away from a provider can drop the proof that made the
194
+ // overlay's hostnames servable, so re-derive it here too.
195
+ await this.opsServerRef.dcRouterRef.resyncPrivateRouteDnsOverlay(
196
+ 'domain migration',
197
+ );
186
198
  this.opsServerRef.invalidateRealtime?.(['dns', 'emailDomains'], {
187
199
  changedIds: [dataArg.id],
188
200
  reason: 'domain-migrated',
@@ -604,7 +604,7 @@ export class GatewayClientHandler {
604
604
  enabled: Boolean(dcRouter.remoteIngressManager?.getHubSettings().enabled),
605
605
  },
606
606
  dns: {
607
- authoritative: Boolean(dcRouter.options.dnsScopes?.length),
607
+ authoritative: Boolean(dcRouter.dnsAuthorityManager?.getEffectiveZoneNames().length),
608
608
  providerManaged: Boolean(dcRouter.dnsManager),
609
609
  },
610
610
  http3: {
@@ -17,6 +17,7 @@ export * from './network-target.handler.js';
17
17
  export * from './users.handler.js';
18
18
  export * from './dns-provider.handler.js';
19
19
  export * from './domain.handler.js';
20
+ export * from './dns-authority.handler.js';
20
21
  export * from './dns-record.handler.js';
21
22
  export * from './acme-config.handler.js';
22
23
  export * from './email-domain.handler.js';
package/ts/readme.md CHANGED
@@ -58,7 +58,7 @@ await router.start();
58
58
  - SmartProxy for HTTP/HTTPS/TCP routes
59
59
  - `UnifiedEmailServer` for SMTP ingress, delivery, managed app address bindings, and outbound SMTP submission when `emailConfig` is present
60
60
  - DB-backed managers for routes, API tokens, target profiles, domains, records, ACME config, and email domains when the DB is enabled
61
- - embedded authoritative DNS and DoH route generation from `dnsNsDomains` and `dnsScopes`
61
+ - embedded authoritative DNS and DoH route generation from `dnsNsDomains` plus the delegation-verified zones held in the database
62
62
  - VPN, RADIUS, and remote ingress services when their config blocks are enabled
63
63
  - OpsServer and the dashboard, which start on every boot
64
64
  - an admin-JWT authenticated read-only MCP endpoint at `/mcp` for safe route, DNS, email, RemoteIngress, and VPN summaries
@@ -3,6 +3,6 @@
3
3
  */
4
4
  export const commitinfo = {
5
5
  name: '@serve.zone/dcrouter',
6
- version: '17.10.2',
6
+ version: '18.0.1',
7
7
  description: 'A multifaceted routing service handling mail and SMS delivery functions.'
8
8
  }