@serve.zone/dcrouter 5.5.0 → 6.2.0

package/ts/classes.dcrouter.ts

@@ -14,6 +14,7 @@ import { logger } from './logger.js';
 // Import storage manager
 import { StorageManager, type IStorageConfig } from './storage/index.js';
 import { StorageBackedCertManager } from './classes.storage-cert-manager.js';
+import { CertProvisionScheduler } from './classes.cert-provision-scheduler.js';
 // Import cache system
 import { CacheDb, CacheCleaner, type ICacheDbOptions } from './cache/index.js';
 
@@ -184,16 +185,19 @@ export class DcRouter {
   public cacheDb?: CacheDb;
   public cacheCleaner?: CacheCleaner;
 
-  // Certificate status tracking from SmartProxy events
+  // Certificate status tracking from SmartProxy events (keyed by domain)
   public certificateStatusMap = new Map<string, {
     status: 'valid' | 'failed';
-    domain: string;
+    routeNames: string[];
     expiryDate?: string;
     issuedAt?: string;
     source?: string;
     error?: string;
   }>();
 
+  // Certificate provisioning scheduler with per-domain backoff
+  public certProvisionScheduler?: CertProvisionScheduler;
+
   // TypedRouter for API endpoints
   public typedrouter = new plugins.typedrequest.TypedRouter();
 
@@ -467,6 +471,9 @@ export class DcRouter {
         },
       };
       
+      // Initialize cert provision scheduler
+      this.certProvisionScheduler = new CertProvisionScheduler(this.storageManager);
+
       // If we have DNS challenge handlers, create SmartAcme and wire to certProvisionFunction
       if (challengeHandlers.length > 0) {
         this.smartAcme = new plugins.smartacme.SmartAcme({
@@ -478,15 +485,25 @@ export class DcRouter {
         });
         await this.smartAcme.start();
 
+        const scheduler = this.certProvisionScheduler;
         smartProxyConfig.certProvisionFunction = async (domain, eventComms) => {
+          // Check backoff before attempting provision
+          if (await scheduler.isInBackoff(domain)) {
+            const info = await scheduler.getBackoffInfo(domain);
+            const msg = `Domain ${domain} is in backoff (${info?.failures} failures), retry after ${info?.retryAfter}`;
+            eventComms.warn(msg);
+            throw new Error(msg);
+          }
+
           try {
+            // smartacme v9 handles concurrency, per-domain dedup, and rate limiting internally
             eventComms.log(`Attempting DNS-01 via SmartAcme for ${domain}`);
             eventComms.setSource('smartacme-dns-01');
             const cert = await this.smartAcme.getCertificateForDomain(domain);
             if (cert.validUntil) {
               eventComms.setExpiryDate(new Date(cert.validUntil));
             }
-            return {
+            const result = {
               id: cert.id,
               domainName: cert.domainName,
               created: cert.created,
@@ -495,7 +512,13 @@ export class DcRouter {
               publicKey: cert.publicKey,
               csr: cert.csr,
             };
+
+            // Success — clear any backoff
+            await scheduler.clearBackoff(domain);
+            return result;
           } catch (err) {
+            // Record failure for backoff tracking
+            await scheduler.recordFailure(domain, err.message);
             eventComms.warn(`SmartAcme DNS-01 failed for ${domain}: ${err.message}, falling back to http-01`);
             return 'http01';
           }
@@ -519,39 +542,34 @@ export class DcRouter {
       });
       
       // Always listen for certificate events — emitted by both ACME and certProvisionFunction paths
+      // Events are keyed by domain for domain-centric certificate tracking
       this.smartProxy.on('certificate-issued', (event: plugins.smartproxy.ICertificateIssuedEvent) => {
         console.log(`[DcRouter] Certificate issued for ${event.domain} via ${event.source}, expires ${event.expiryDate}`);
-        const routeName = this.findRouteNameForDomain(event.domain);
-        if (routeName) {
-          this.certificateStatusMap.set(routeName, {
-            status: 'valid', domain: event.domain,
-            expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
-            source: event.source,
-          });
-        }
+        const routeNames = this.findRouteNamesForDomain(event.domain);
+        this.certificateStatusMap.set(event.domain, {
+          status: 'valid', routeNames,
+          expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
+          source: event.source,
+        });
       });
 
       this.smartProxy.on('certificate-renewed', (event: plugins.smartproxy.ICertificateIssuedEvent) => {
         console.log(`[DcRouter] Certificate renewed for ${event.domain} via ${event.source}, expires ${event.expiryDate}`);
-        const routeName = this.findRouteNameForDomain(event.domain);
-        if (routeName) {
-          this.certificateStatusMap.set(routeName, {
-            status: 'valid', domain: event.domain,
-            expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
-            source: event.source,
-          });
-        }
+        const routeNames = this.findRouteNamesForDomain(event.domain);
+        this.certificateStatusMap.set(event.domain, {
+          status: 'valid', routeNames,
+          expiryDate: event.expiryDate, issuedAt: new Date().toISOString(),
+          source: event.source,
+        });
       });
 
       this.smartProxy.on('certificate-failed', (event: plugins.smartproxy.ICertificateFailedEvent) => {
         console.error(`[DcRouter] Certificate failed for ${event.domain} (${event.source}):`, event.error);
-        const routeName = this.findRouteNameForDomain(event.domain);
-        if (routeName) {
-          this.certificateStatusMap.set(routeName, {
-            status: 'failed', domain: event.domain, error: event.error,
-            source: event.source,
-          });
-        }
+        const routeNames = this.findRouteNamesForDomain(event.domain);
+        this.certificateStatusMap.set(event.domain, {
+          status: 'failed', routeNames, error: event.error,
+          source: event.source,
+        });
       });
       
       // Start SmartProxy
@@ -724,7 +742,7 @@ export class DcRouter {
   }
 
   /**
-   * Find the route name that matches a given domain
+   * Find the first route name that matches a given domain
    */
   private findRouteNameForDomain(domain: string): string | undefined {
     if (!this.smartProxy) return undefined;
@@ -740,6 +758,27 @@ export class DcRouter {
     return undefined;
   }
 
+  /**
+   * Find ALL route names that match a given domain
+   */
+  public findRouteNamesForDomain(domain: string): string[] {
+    if (!this.smartProxy) return [];
+    const names: string[] = [];
+    for (const route of this.smartProxy.routeManager.getRoutes()) {
+      if (!route.match.domains || !route.name) continue;
+      const routeDomains = Array.isArray(route.match.domains)
+        ? route.match.domains
+        : [route.match.domains];
+      for (const pattern of routeDomains) {
+        if (this.isDomainMatch(domain, pattern)) {
+          names.push(route.name);
+          break; // This route already matched, no need to check other patterns
+        }
+      }
+    }
+    return names;
+  }
+
   public async stop() {
     console.log('Stopping DcRouter services...');
 

package/ts/opsserver/handlers/certificate.handler.ts

@@ -23,24 +23,45 @@ export class CertificateHandler {
       )
     );
 
-    // Reprovision Certificate
+    // Legacy route-based reprovision (backward compat)
     this.typedrouter.addTypedHandler(
       new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_ReprovisionCertificate>(
         'reprovisionCertificate',
         async (dataArg) => {
-          return this.reprovisionCertificate(dataArg.routeName);
+          return this.reprovisionCertificateByRoute(dataArg.routeName);
+        }
+      )
+    );
+
+    // Domain-based reprovision (preferred)
+    this.typedrouter.addTypedHandler(
+      new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_ReprovisionCertificateDomain>(
+        'reprovisionCertificateDomain',
+        async (dataArg) => {
+          return this.reprovisionCertificateDomain(dataArg.domain);
         }
       )
     );
   }
 
+  /**
+   * Build domain-centric certificate overview.
+   * Instead of one row per route, we produce one row per unique domain.
+   */
   private async buildCertificateOverview(): Promise<interfaces.requests.ICertificateInfo[]> {
     const dcRouter = this.opsServerRef.dcRouterRef;
     const smartProxy = dcRouter.smartProxy;
     if (!smartProxy) return [];
 
     const routes = smartProxy.routeManager.getRoutes();
-    const certificates: interfaces.requests.ICertificateInfo[] = [];
+
+    // Phase 1: Collect unique domains with their associated route info
+    const domainMap = new Map<string, {
+      routeNames: string[];
+      source: interfaces.requests.TCertificateSource;
+      tlsMode: 'terminate' | 'terminate-and-reencrypt' | 'passthrough';
+      canReprovision: boolean;
+    }>();
 
     for (const route of routes) {
       if (!route.name) continue;
@@ -58,7 +79,6 @@ export class CertificateHandler {
       // Determine source
       let source: interfaces.requests.TCertificateSource = 'none';
       if (tls.certificate === 'auto') {
-        // Check if a certProvisionFunction is configured
         if ((smartProxy.settings as any).certProvisionFunction) {
           source = 'provision-function';
         } else {
@@ -68,15 +88,44 @@ export class CertificateHandler {
         source = 'static';
       }
 
-      // Start with unknown status
+      const canReprovision = source === 'acme' || source === 'provision-function';
+      const tlsMode = tls.mode as 'terminate' | 'terminate-and-reencrypt' | 'passthrough';
+
+      for (const domain of routeDomains) {
+        const existing = domainMap.get(domain);
+        if (existing) {
+          // Add this route name to the existing domain entry
+          if (!existing.routeNames.includes(route.name)) {
+            existing.routeNames.push(route.name);
+          }
+          // Upgrade source if more specific
+          if (existing.source === 'none' && source !== 'none') {
+            existing.source = source;
+            existing.canReprovision = canReprovision;
+          }
+        } else {
+          domainMap.set(domain, {
+            routeNames: [route.name],
+            source,
+            tlsMode,
+            canReprovision,
+          });
+        }
+      }
+    }
+
+    // Phase 2: Resolve status for each unique domain
+    const certificates: interfaces.requests.ICertificateInfo[] = [];
+
+    for (const [domain, info] of domainMap) {
       let status: interfaces.requests.TCertificateStatus = 'unknown';
       let expiryDate: string | undefined;
       let issuedAt: string | undefined;
       let issuer: string | undefined;
       let error: string | undefined;
 
-      // Check event-based status from DcRouter's certificateStatusMap
-      const eventStatus = dcRouter.certificateStatusMap.get(route.name);
+      // Check event-based status from certificateStatusMap (now keyed by domain)
+      const eventStatus = dcRouter.certificateStatusMap.get(domain);
       if (eventStatus) {
         status = eventStatus.status;
         expiryDate = eventStatus.expiryDate;
@@ -87,10 +136,10 @@ export class CertificateHandler {
         }
       }
 
-      // Try Rust-side certificate status if no event data
-      if (status === 'unknown') {
+      // Try SmartProxy certificate status if no event data
+      if (status === 'unknown' && info.routeNames.length > 0) {
         try {
-          const rustStatus = await smartProxy.getCertificateStatus(route.name);
+          const rustStatus = await smartProxy.getCertificateStatus(info.routeNames[0]);
           if (rustStatus) {
             if (rustStatus.expiryDate) expiryDate = rustStatus.expiryDate;
             if (rustStatus.issuer) issuer = rustStatus.issuer;
@@ -105,22 +154,19 @@ export class CertificateHandler {
       }
 
       // Check persisted cert data from StorageManager
-      if (status === 'unknown' && routeDomains.length > 0) {
-        for (const domain of routeDomains) {
-          if (expiryDate) break;
-          const cleanDomain = domain.replace(/^\*\.?/, '');
-          const certData = await dcRouter.storageManager.getJSON(`/certs/${cleanDomain}`);
-          if (certData?.validUntil) {
-            expiryDate = new Date(certData.validUntil).toISOString();
-            if (certData.created) {
-              issuedAt = new Date(certData.created).toISOString();
-            }
-            issuer = 'smartacme-dns-01';
+      if (status === 'unknown') {
+        const cleanDomain = domain.replace(/^\*\.?/, '');
+        const certData = await dcRouter.storageManager.getJSON(`/certs/${cleanDomain}`);
+        if (certData?.validUntil) {
+          expiryDate = new Date(certData.validUntil).toISOString();
+          if (certData.created) {
+            issuedAt = new Date(certData.created).toISOString();
           }
+          issuer = 'smartacme-dns-01';
         }
       }
 
-      // Compute status from expiry date if we have one and status is still valid/unknown
+      // Compute status from expiry date
       if (expiryDate && (status === 'valid' || status === 'unknown')) {
         const expiry = new Date(expiryDate);
         const now = new Date();
@@ -136,28 +182,36 @@ export class CertificateHandler {
       }
 
       // Static certs with no other info default to 'valid'
-      if (source === 'static' && status === 'unknown') {
+      if (info.source === 'static' && status === 'unknown') {
         status = 'valid';
       }
 
       // ACME/provision-function routes with no cert data are still provisioning
-      if (status === 'unknown' && (source === 'acme' || source === 'provision-function')) {
+      if (status === 'unknown' && (info.source === 'acme' || info.source === 'provision-function')) {
         status = 'provisioning';
       }
 
-      const canReprovision = source === 'acme' || source === 'provision-function';
+      // Phase 3: Attach backoff info
+      let backoffInfo: interfaces.requests.ICertificateInfo['backoffInfo'];
+      if (dcRouter.certProvisionScheduler) {
+        const bi = await dcRouter.certProvisionScheduler.getBackoffInfo(domain);
+        if (bi) {
+          backoffInfo = bi;
+        }
+      }
 
       certificates.push({
-        routeName: route.name,
-        domains: routeDomains,
+        domain,
+        routeNames: info.routeNames,
         status,
-        source,
-        tlsMode: tls.mode as 'terminate' | 'terminate-and-reencrypt' | 'passthrough',
+        source: info.source,
+        tlsMode: info.tlsMode,
         expiryDate,
         issuer,
         issuedAt,
         error,
-        canReprovision,
+        canReprovision: info.canReprovision,
+        backoffInfo,
       });
     }
 
@@ -187,7 +241,10 @@ export class CertificateHandler {
     return summary;
   }
 
-  private async reprovisionCertificate(routeName: string): Promise<{ success: boolean; message?: string }> {
+  /**
+   * Legacy route-based reprovisioning
+   */
+  private async reprovisionCertificateByRoute(routeName: string): Promise<{ success: boolean; message?: string }> {
     const dcRouter = this.opsServerRef.dcRouterRef;
     const smartProxy = dcRouter.smartProxy;
 
@@ -197,11 +254,58 @@ export class CertificateHandler {
 
     try {
       await smartProxy.provisionCertificate(routeName);
-      // Clear event-based status so it gets refreshed
-      dcRouter.certificateStatusMap.delete(routeName);
+      // Clear event-based status for domains in this route
+      for (const [domain, entry] of dcRouter.certificateStatusMap) {
+        if (entry.routeNames.includes(routeName)) {
+          dcRouter.certificateStatusMap.delete(domain);
+        }
+      }
       return { success: true, message: `Certificate reprovisioning triggered for route '${routeName}'` };
     } catch (err) {
       return { success: false, message: err.message || 'Failed to reprovision certificate' };
     }
   }
+
+  /**
+   * Domain-based reprovisioning — clears backoff first, then triggers provision
+   */
+  private async reprovisionCertificateDomain(domain: string): Promise<{ success: boolean; message?: string }> {
+    const dcRouter = this.opsServerRef.dcRouterRef;
+    const smartProxy = dcRouter.smartProxy;
+
+    if (!smartProxy) {
+      return { success: false, message: 'SmartProxy is not running' };
+    }
+
+    // Clear backoff for this domain (user override)
+    if (dcRouter.certProvisionScheduler) {
+      await dcRouter.certProvisionScheduler.clearBackoff(domain);
+    }
+
+    // Clear status map entry so it gets refreshed
+    dcRouter.certificateStatusMap.delete(domain);
+
+    // Try to provision via SmartAcme directly
+    if (dcRouter.smartAcme) {
+      try {
+        await dcRouter.smartAcme.getCertificateForDomain(domain);
+        return { success: true, message: `Certificate reprovisioning triggered for domain '${domain}'` };
+      } catch (err) {
+        return { success: false, message: err.message || `Failed to reprovision certificate for ${domain}` };
+      }
+    }
+
+    // Fallback: try provisioning via the first matching route
+    const routeNames = dcRouter.findRouteNamesForDomain(domain);
+    if (routeNames.length > 0) {
+      try {
+        await smartProxy.provisionCertificate(routeNames[0]);
+        return { success: true, message: `Certificate reprovisioning triggered for domain '${domain}' via route '${routeNames[0]}'` };
+      } catch (err) {
+        return { success: false, message: err.message || `Failed to reprovision certificate for ${domain}` };
+      }
+    }
+
+    return { success: false, message: `No routes found for domain '${domain}'` };
+  }
 }

package/ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '5.5.0',
+  version: '6.2.0',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }

package/ts_web/appstate.ts

@@ -719,18 +719,18 @@ export const fetchCertificateOverviewAction = certificateStatePart.createAction(
 });
 
 export const reprovisionCertificateAction = certificateStatePart.createAction<string>(
-  async (statePartArg, routeName) => {
+  async (statePartArg, domain) => {
     const context = getActionContext();
     const currentState = statePartArg.getState();
 
     try {
       const request = new plugins.domtools.plugins.typedrequest.TypedRequest<
-        interfaces.requests.IReq_ReprovisionCertificate
-      >('/typedrequest', 'reprovisionCertificate');
+        interfaces.requests.IReq_ReprovisionCertificateDomain
+      >('/typedrequest', 'reprovisionCertificateDomain');
 
       await request.fire({
         identity: context.identity,
-        routeName,
+        domain,
       });
 
       // Re-fetch overview after reprovisioning

package/ts_web/elements/ops-view-certificates.ts

@@ -94,13 +94,13 @@ export class OpsViewCertificates extends DeesElement {
         color: ${cssManager.bdTheme('#374151', '#d1d5db')};
       }
 
-      .domainPills {
+      .routePills {
         display: flex;
         flex-wrap: wrap;
         gap: 4px;
       }
 
-      .domainPill {
+      .routePill {
         display: inline-flex;
         align-items: center;
         padding: 2px 8px;
@@ -125,6 +125,17 @@ export class OpsViewCertificates extends DeesElement {
         white-space: nowrap;
       }
 
+      .backoffIndicator {
+        display: inline-flex;
+        align-items: center;
+        gap: 4px;
+        font-size: 11px;
+        color: ${cssManager.bdTheme('#9a3412', '#fb923c')};
+        padding: 2px 6px;
+        border-radius: 4px;
+        background: ${cssManager.bdTheme('#fff7ed', '#431407')};
+      }
+
       .expiryInfo {
         font-size: 12px;
       }
@@ -218,14 +229,16 @@ export class OpsViewCertificates extends DeesElement {
       <dees-table
         .data=${this.certState.certificates}
         .displayFunction=${(cert: interfaces.requests.ICertificateInfo) => ({
-          Route: cert.routeName,
-          Domains: this.renderDomainPills(cert.domains),
+          Domain: cert.domain,
+          Routes: this.renderRoutePills(cert.routeNames),
           Status: this.renderStatusBadge(cert.status),
           Source: this.renderSourceBadge(cert.source),
           Expires: this.renderExpiry(cert.expiryDate),
-          Error: cert.error
-            ? html`<span class="errorText" title="${cert.error}">${cert.error}</span>`
-            : '',
+          Error: cert.backoffInfo
+            ? html`<span class="backoffIndicator">${cert.backoffInfo.failures} failures, retry ${this.formatRetryTime(cert.backoffInfo.retryAfter)}</span>`
+            : cert.error
+              ? html`<span class="errorText" title="${cert.error}">${cert.error}</span>`
+              : '',
         })}
         .dataActions=${[
           {
@@ -245,11 +258,11 @@ export class OpsViewCertificates extends DeesElement {
               }
               await appstate.certificateStatePart.dispatchAction(
                 appstate.reprovisionCertificateAction,
-                cert.routeName,
+                cert.domain,
               );
               const { DeesToast } = await import('@design.estate/dees-catalog');
               DeesToast.show({
-                message: `Reprovisioning triggered for ${cert.routeName}`,
+                message: `Reprovisioning triggered for ${cert.domain}`,
                 type: 'success',
                 duration: 3000,
               });
@@ -263,7 +276,7 @@ export class OpsViewCertificates extends DeesElement {
               const cert = actionData.item;
               const { DeesModal } = await import('@design.estate/dees-catalog');
               await DeesModal.createAndShow({
-                heading: `Certificate: ${cert.routeName}`,
+                heading: `Certificate: ${cert.domain}`,
                 content: html`
                   <div style="padding: 20px;">
                     <dees-dataview-codebox
@@ -275,10 +288,10 @@ export class OpsViewCertificates extends DeesElement {
                 `,
                 menuOptions: [
                   {
-                    name: 'Copy Route Name',
+                    name: 'Copy Domain',
                     iconName: 'copy',
                     action: async () => {
-                      await navigator.clipboard.writeText(cert.routeName);
+                      await navigator.clipboard.writeText(cert.domain);
                     },
                   },
                 ],
@@ -287,7 +300,7 @@ export class OpsViewCertificates extends DeesElement {
           },
         ]}
         heading1="Certificate Status"
-        heading2="TLS certificates across all routes"
+        heading2="TLS certificates by domain"
         searchable
         .pagination=${true}
         .paginationSize=${50}
@@ -296,14 +309,14 @@ export class OpsViewCertificates extends DeesElement {
     `;
   }
 
-  private renderDomainPills(domains: string[]): TemplateResult {
+  private renderRoutePills(routeNames: string[]): TemplateResult {
     const maxShow = 3;
-    const visible = domains.slice(0, maxShow);
-    const remaining = domains.length - maxShow;
+    const visible = routeNames.slice(0, maxShow);
+    const remaining = routeNames.length - maxShow;
 
     return html`
-      <span class="domainPills">
-        ${visible.map((d) => html`<span class="domainPill">${d}</span>`)}
+      <span class="routePills">
+        ${visible.map((r) => html`<span class="routePill">${r}</span>`)}
         ${remaining > 0 ? html`<span class="moreCount">+${remaining} more</span>` : ''}
       </span>
     `;
@@ -352,4 +365,16 @@ export class OpsViewCertificates extends DeesElement {
       </span>
     `;
   }
+
+  private formatRetryTime(retryAfter?: string): string {
+    if (!retryAfter) return 'soon';
+    const retryDate = new Date(retryAfter);
+    const now = new Date();
+    const diffMs = retryDate.getTime() - now.getTime();
+    if (diffMs <= 0) return 'now';
+    const diffMin = Math.ceil(diffMs / 60000);
+    if (diffMin < 60) return `in ${diffMin}m`;
+    const diffHours = Math.ceil(diffMin / 60);
+    return `in ${diffHours}h`;
+  }
 }

package/ts_web/readme.md

@@ -34,6 +34,13 @@ For reporting bugs, issues, or security vulnerabilities, please visit [community
 - **Security** — Security incidents from email processing
 - Bounce record management and suppression list controls
 
+### 🔐 Certificate Management
+- Domain-centric certificate overview with status indicators
+- Certificate source tracking (ACME, provision function, static)
+- Expiry date monitoring and alerts
+- Per-domain backoff status for failed provisions
+- One-click reprovisioning per domain
+
 ### 📜 Log Viewer
 - Real-time log streaming
 - Filter by log level (error, warning, info, debug)
@@ -77,6 +84,7 @@ ts_web/
     ├── ops-view-overview.ts  # Overview statistics
     ├── ops-view-network.ts   # Network monitoring
     ├── ops-view-emails.ts    # Email queue management
+    ├── ops-view-certificates.ts  # Certificate overview & reprovisioning
     ├── ops-view-logs.ts      # Log viewer
     ├── ops-view-config.ts    # Configuration display
     ├── ops-view-security.ts  # Security dashboard
@@ -132,6 +140,7 @@ removeFromSuppressionAction(email) // Remove from suppression list
   /emails/sent      → Sent emails
   /emails/failed    → Failed emails
   /emails/security  → Security incidents
+/certificates    → Certificate management
 /logs            → Log viewer
 /configuration   → System configuration
 /security        → Security dashboard