@serve.zone/dcrouter 17.10.2 → 18.0.0

package/ts/acme/acme-failure-classification.ts

@@ -0,0 +1,201 @@
+import { PlatformError, type IErrorContext } from '../errors/base.errors.js';
+import {
+  DCR_ACME_PERMANENT_FAILURE,
+  ErrorCategory,
+  ErrorRecoverability,
+  ErrorSeverity,
+} from '../errors/error.codes.js';
+import { DomainOwnershipError } from '../dns/domain-ownership.js';
+
+/**
+ * ACME failure classification.
+ *
+ * dcrouter has two independent ACME retry budgets, and neither used to look at
+ * *why* a failure happened:
+ *
+ *  1. `SmartAcmeLifecycle` — SmartAcme provider startup. 5 s→1 h exponential
+ *     backoff with jitter, hard cap of 20 attempts, then permanent give-up.
+ *     Only a SmartProxy rebuild or a process restart re-arms it.
+ *  2. `CertProvisionScheduler` — per-domain certificate provisioning.
+ *     `min(failures², 24 h)` backoff, **no cap**, re-armed by time forever.
+ *     This is the budget that reached 31–45 failures on the broken domains.
+ *
+ * Retrying is correct for rate limits, DNS propagation and transport faults. It
+ * is never correct for a configuration cause: a hostname with no managed domain
+ * cannot acquire one by waiting, so every one of those attempts was a silent
+ * no-op that also kept the real reason out of the operator's view. Permanent
+ * causes must therefore skip the budget entirely and surface attributably.
+ *
+ * Unclassified causes stay transient on purpose. Guessing "permanent" would
+ * strand recoverable domains, so the default preserves existing retry behaviour;
+ * only causes we can positively recognise are treated as terminal.
+ */
+
+export type TAcmeFailureReason =
+  /** Ownership of the hostname could not be proven (no DomainDoc / no provider zone / not delegation-verified). */
+  | 'domain-ownership-unverified'
+  /** The DNS-01 dispatcher found no managed zone able to hold the challenge record. */
+  | 'no-managed-dns-zone'
+  /** No challenge handler / provider is wired for this domain at all. */
+  | 'no-challenge-handler'
+  /** The ACME account itself is misconfigured (email, terms, key, directory URL). */
+  | 'acme-account-configuration'
+  /** CAA forbids our issuer — only the domain holder can change this. */
+  | 'caa-forbids-issuance'
+  /** ACME server rate limit — retrying is exactly right. */
+  | 'rate-limited'
+  /** Challenge not yet visible, propagation delay, transport fault. */
+  | 'transient'
+  /** Nothing recognised. Treated as transient so recoverable causes keep retrying. */
+  | 'unclassified';
+
+export interface IAcmeFailureClassification {
+  reason: TAcmeFailureReason;
+  /** True when no retry can resolve the cause, so the retry budget must not be consumed. */
+  permanent: boolean;
+  message: string;
+}
+
+interface IReasonPattern {
+  reason: TAcmeFailureReason;
+  permanent: boolean;
+  patterns: RegExp[];
+}
+
+/**
+ * Ordered most-specific-first. Each pattern must only match text that uniquely
+ * identifies the cause — a false "permanent" verdict silently stops legitimate
+ * retries, which is a worse failure than an extra retry.
+ */
+const reasonPatterns: IReasonPattern[] = [
+  {
+    // Thrown by DnsManager.buildAcmeConvenientDnsProvider() when no DomainDoc covers the FQDN.
+    reason: 'no-managed-dns-zone',
+    permanent: true,
+    patterns: [
+      /no managed domain found for/i,
+      /add the domain in domains before issuing certificates/i,
+    ],
+  },
+  {
+    reason: 'no-challenge-handler',
+    permanent: true,
+    patterns: [
+      /no (?:challenge )?handler (?:found |available )?for/i,
+      /no dns-01 (?:handler|provider)/i,
+      /domain is not supported by any challenge handler/i,
+    ],
+  },
+  {
+    reason: 'caa-forbids-issuance',
+    permanent: true,
+    patterns: [
+      /caa record(?:s)? (?:for [^\s]+ )?(?:prevent|forbid|do not allow)/i,
+      /urn:ietf:params:acme:error:caa/i,
+    ],
+  },
+  {
+    reason: 'acme-account-configuration',
+    permanent: true,
+    patterns: [
+      /urn:ietf:params:acme:error:invalidemail/i,
+      /urn:ietf:params:acme:error:accountdoesnotexist/i,
+      /urn:ietf:params:acme:error:unsupportedcontact/i,
+      /must agree to (?:the )?terms of service/i,
+      /accountemail is required/i,
+    ],
+  },
+  {
+    reason: 'rate-limited',
+    permanent: false,
+    patterns: [
+      /urn:ietf:params:acme:error:ratelimited/i,
+      /too many (?:certificates|requests|failed authorizations)/i,
+      /rate ?limit/i,
+    ],
+  },
+];
+
+const extractMessage = (errorArg: unknown): string => {
+  if (errorArg instanceof Error) return errorArg.message;
+  if (typeof errorArg === 'string') return errorArg;
+  if (errorArg && typeof errorArg === 'object' && 'message' in errorArg) {
+    return String((errorArg as { message: unknown }).message);
+  }
+  return String(errorArg);
+};
+
+/**
+ * Classify an ACME/provisioning failure into a retryable or terminal cause.
+ * Structured errors win over text matching: a `DomainOwnershipError` and any
+ * NON_RECOVERABLE CONFIGURATION `PlatformError` are permanent by declaration.
+ */
+export const classifyAcmeFailure = (errorArg: unknown): IAcmeFailureClassification => {
+  const message = extractMessage(errorArg);
+
+  if (errorArg instanceof DomainOwnershipError) {
+    return { reason: 'domain-ownership-unverified', permanent: true, message };
+  }
+
+  if (
+    errorArg instanceof PlatformError
+    && errorArg.category === ErrorCategory.CONFIGURATION
+    && errorArg.recoverability === ErrorRecoverability.NON_RECOVERABLE
+  ) {
+    return { reason: 'acme-account-configuration', permanent: true, message };
+  }
+
+  for (const candidate of reasonPatterns) {
+    if (candidate.patterns.some((pattern) => pattern.test(message))) {
+      return { reason: candidate.reason, permanent: candidate.permanent, message };
+    }
+  }
+
+  return { reason: 'unclassified', permanent: false, message };
+};
+
+/**
+ * Terminal ACME failure. HIGH severity so the automatic PlatformError log lands
+ * at `error` rather than being lost in provisioning warn noise, and
+ * NON_RECOVERABLE so `isRetryable()` and every downstream retry layer agree that
+ * this must not be retried.
+ */
+export class AcmePermanentFailureError extends PlatformError {
+  public readonly classification: IAcmeFailureClassification;
+
+  constructor(
+    classification: IAcmeFailureClassification,
+    operation: string,
+    component: string,
+    context: IErrorContext = {},
+  ) {
+    super(
+      `${operation} failed permanently (${classification.reason}): ${classification.message}`,
+      DCR_ACME_PERMANENT_FAILURE,
+      ErrorSeverity.HIGH,
+      ErrorCategory.CONFIGURATION,
+      ErrorRecoverability.NON_RECOVERABLE,
+      {
+        component,
+        operation,
+        userMessage:
+          `${operation} cannot succeed until its configuration is fixed (${classification.reason}): ${classification.message}`,
+        ...context,
+        data: {
+          acmeFailureReason: classification.reason,
+          ...context.data,
+        },
+      },
+    );
+    this.classification = classification;
+  }
+
+  protected createWithContext(context: IErrorContext): PlatformError {
+    return new AcmePermanentFailureError(
+      this.classification,
+      this.context.operation || 'operation',
+      this.context.component || 'acme',
+      context,
+    );
+  }
+}

package/ts/acme/classes.smartacme-lifecycle.ts

@@ -1,18 +1,58 @@
 import * as plugins from '../plugins.js';
 import { logger } from '../logger.js';
 import type { DcRouter } from '../classes.dcrouter.js';
+import {
+  AcmePermanentFailureError,
+  classifyAcmeFailure,
+  type TAcmeFailureReason,
+} from './acme-failure-classification.js';
+
+/** Terminal state of the SmartAcme provider startup budget. */
+export interface IAcmeStartFailure {
+  reason: TAcmeFailureReason;
+  /** True when no further retry can resolve the cause. */
+  permanent: boolean;
+  /** Attempts consumed when the budget ended. */
+  attempts: number;
+  message: string;
+  at: number;
+  /** What an operator has to do to re-arm startup. */
+  rearmedBy: string;
+}
+
+/** Attempts allowed for transient SmartAcme startup failures before giving up. */
+export const smartAcmeStartMaxAttempts = 20;
 
 /**
  * Background start/retry/stop lifecycle for the DcRouter-owned SmartAcme
  * instance. SmartAcme startup can hit ACME rate limits, so startup runs in
  * the background with generation-guarded exponential retry, and certificate
  * provisioning is re-triggered once DNS-01 becomes ready.
+ *
+ * Retry semantics, which are NOT shared with the per-domain
+ * `CertProvisionScheduler` budget:
+ *  - increments once per failed `smartAcme.start()`;
+ *  - 5 s → 1 h exponential backoff with ±20% jitter;
+ *  - capped at `smartAcmeStartMaxAttempts` transient attempts;
+ *  - reset by `startInBackground()`, `stop()`, and a successful start;
+ *  - re-armed only by `startInBackground()`, i.e. a SmartProxy rebuild, an
+ *    explicit `rearm()` after a configuration change, or a process restart.
+ *    Nothing re-arms it on a timer, so the terminal state is recorded in
+ *    `startFailure` and logged at `error` instead of scrolling past as a warning.
+ *
+ * A permanent cause never enters the backoff at all: retrying a misconfigured
+ * ACME account cannot fix it, and spending the budget on it hides the reason.
  */
 export class SmartAcmeLifecycle {
   /** True once the SmartAcme DNS-01 provider finished starting. */
   public ready = false;
   /** Tracks whether the taskbuffer SmartAcme service is started, so SmartProxy rebuilds can re-kick startup. */
   public serviceStarted = false;
+  /**
+   * Set when the startup budget ended — either immediately for a permanent cause
+   * or after the transient attempt cap. Cleared on every (re-)arm and on success.
+   */
+  public startFailure?: IAcmeStartFailure;
 
   private startGeneration = 0;
   private startPromise?: Promise<void>;
@@ -30,14 +70,35 @@ export class SmartAcmeLifecycle {
     const generation = ++this.startGeneration;
     this.ready = false;
     this.retryAttempt = 0;
+    this.startFailure = undefined;
     this.clearRetryTimer();
     this.scheduleStart(generation, 0);
   }
 
+  /**
+   * Re-arm startup after a configuration change that can plausibly fix a
+   * previously terminal cause (e.g. the ACME account settings were corrected).
+   * Without this, an exhausted or permanently-failed budget could only be reset
+   * by a SmartProxy rebuild or a full restart — and a dcrouter restart is a
+   * measured 30–60 s of total public outage.
+   */
+  public rearm(reasonArg: string): boolean {
+    if (!this.dcRouterRef.smartAcme || !this.serviceStarted) {
+      return false;
+    }
+    if (this.ready && !this.startFailure) {
+      return false;
+    }
+    logger.log('info', `Re-arming SmartAcme DNS-01 provider startup: ${reasonArg}`);
+    this.startInBackground();
+    return true;
+  }
+
   public async stop(): Promise<void> {
     this.startGeneration++;
     this.ready = false;
     this.retryAttempt = 0;
+    this.startFailure = undefined;
     this.clearRetryTimer();
 
     const smartAcme = this.dcRouterRef.smartAcme;
@@ -91,6 +152,7 @@ export class SmartAcmeLifecycle {
 
       this.ready = true;
       this.retryAttempt = 0;
+      this.startFailure = undefined;
       logger.log('info', 'SmartAcme DNS-01 provider is now ready');
       this.retriggerCertificateProvisioning();
     } catch (err) {
@@ -102,9 +164,48 @@ export class SmartAcmeLifecycle {
       await smartAcme.stop().catch((stopErr) => {
         logger.log('warn', `Failed to clean up SmartAcme after startup failure: ${(stopErr as Error).message}`);
       });
+
+      // Classify before spending any of the budget: a permanent cause is not made
+      // truer by 20 more attempts, and every attempt spent on it is an hour in
+      // which nothing tells the operator what is actually wrong.
+      const classification = classifyAcmeFailure(err);
+      if (classification.permanent) {
+        this.startFailure = {
+          reason: classification.reason,
+          permanent: true,
+          attempts: this.retryAttempt,
+          message: classification.message,
+          at: Date.now(),
+          rearmedBy: 'fixing the ACME configuration (which calls rearm()), a SmartProxy rebuild, or a process restart',
+        };
+        // Constructing this logs at `error` with the code and category attached.
+        new AcmePermanentFailureError(
+          classification,
+          'SmartAcme DNS-01 provider startup',
+          'smartacme-lifecycle',
+          { data: { attempts: this.retryAttempt, rearmedBy: this.startFailure.rearmedBy } },
+        );
+        return;
+      }
+
       this.retryAttempt++;
-      if (this.retryAttempt > 20) {
-        logger.log('error', `SmartAcme DNS-01 provider failed after 20 startup attempts: ${(err as Error).message}`);
+      if (this.retryAttempt > smartAcmeStartMaxAttempts) {
+        this.startFailure = {
+          reason: classification.reason,
+          permanent: false,
+          attempts: this.retryAttempt - 1,
+          message: classification.message,
+          at: Date.now(),
+          rearmedBy: 'a SmartProxy rebuild, an explicit rearm() after a configuration change, or a process restart',
+        };
+        logger.log(
+          'error',
+          `SmartAcme DNS-01 provider gave up after ${smartAcmeStartMaxAttempts} startup attempts `
+            + `(${classification.reason}): ${classification.message}. `
+            + 'Nothing re-arms this on a timer — DNS-01 stays unavailable and every affected route falls back to a '
+            + `no-op http-01 path until ${this.startFailure.rearmedBy}.`,
+          { acmeFailureReason: classification.reason, attempts: this.startFailure.attempts },
+        );
         return;
       }
 
@@ -113,7 +214,7 @@ export class SmartAcmeLifecycle {
       const delayMs = Math.min(baseDelayMs * Math.pow(2, this.retryAttempt - 1), maxDelayMs);
       const jitter = 0.8 + Math.random() * 0.4;
       const actualDelayMs = Math.floor(delayMs * jitter);
-      logger.log('warn', `SmartAcme DNS-01 provider startup failed: ${(err as Error).message}; retrying in ${actualDelayMs}ms (attempt ${this.retryAttempt}/20)`);
+      logger.log('warn', `SmartAcme DNS-01 provider startup failed: ${(err as Error).message}; retrying in ${actualDelayMs}ms (attempt ${this.retryAttempt}/${smartAcmeStartMaxAttempts})`);
       this.scheduleStart(generation, actualDelayMs);
     } finally {
       if (this.startPromise === startPromise) {

package/ts/acme/index.ts

@@ -1,2 +1,3 @@
 export * from './manager.acme-config.js';
 export * from './classes.smartacme-lifecycle.js';
+export * from './acme-failure-classification.js';