@serve.zone/dcrouter 13.12.0 → 13.14.0

package/ts/dns/manager.dns.ts

@@ -296,70 +296,99 @@ export class DnsManager {
   }
 
   /**
-   * True if any cloudflare provider exists in the DB. Used by setupSmartProxy()
-   * to decide whether to wire SmartAcme with a DNS-01 handler.
+   * Find the DomainDoc that covers a given FQDN, regardless of source
+   * (dcrouter-hosted or provider-managed). Uses longest-suffix match.
    */
-  public async hasAcmeCapableProvider(): Promise<boolean> {
-    const providers = await DnsProviderDoc.findAll();
-    return providers.length > 0;
+  public async findDomainForFqdn(fqdn: string): Promise<DomainDoc | null> {
+    const lower = fqdn.toLowerCase().replace(/^\*\./, '').replace(/\.$/, '');
+    const allDomains = await DomainDoc.findAll();
+    // Sort by name length descending for longest-match-wins
+    allDomains.sort((a, b) => b.name.length - a.name.length);
+    for (const domain of allDomains) {
+      if (lower === domain.name || lower.endsWith(`.${domain.name}`)) {
+        return domain;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Delete all DNS records matching a name and type under a domain.
+   * Used for ACME challenge cleanup (may have multiple TXT records at the same name).
+   */
+  public async deleteRecordsByNameAndType(
+    domainId: string,
+    name: string,
+    type: TDnsRecordType,
+  ): Promise<void> {
+    const records = await DnsRecordDoc.findByDomainId(domainId);
+    for (const rec of records) {
+      if (rec.name.toLowerCase() === name.toLowerCase() && rec.type === type) {
+        await this.deleteRecord(rec.id);
+      }
+    }
+  }
+
+  /**
+   * True if any domain is under management (dcrouter-hosted or provider-managed).
+   * Used by setupSmartProxy() to decide whether to wire SmartAcme with a DNS-01 handler.
+   */
+  public async hasAnyManagedDomain(): Promise<boolean> {
+    const domains = await DomainDoc.findAll();
+    return domains.length > 0;
   }
 
   /**
-   * Build an IConvenientDnsProvider that dispatches each ACME challenge to
-   * the right provider client (whichever provider type owns the parent zone),
-   * based on the challenge's hostName. Provider-agnostic — uses the IDnsProviderClient
-   * interface, so any registered provider implementation works.
-   * Returned object plugs directly into smartacme's Dns01Handler.
+   * Build an IConvenientDnsProvider that routes ACME DNS-01 challenges through
+   * the DnsManager abstraction. Challenges are dispatched via createRecord() /
+   * deleteRecord(), which transparently handle both dcrouter-hosted zones
+   * (embedded DnsServer) and provider-managed zones (e.g. Cloudflare API).
+   *
+   * Only domains under management (with a DomainDoc in DB) are supported —
+   * this acts as the management gate for certificate issuance.
    */
   public buildAcmeConvenientDnsProvider(): plugins.tsclass.network.IConvenientDnsProvider {
     const self = this;
     const adapter = {
       async acmeSetDnsChallenge(dnsChallenge: { hostName: string; challenge: string }) {
-        const client = await self.getProviderClientForDomain(dnsChallenge.hostName);
-        if (!client) {
+        const domainDoc = await self.findDomainForFqdn(dnsChallenge.hostName);
+        if (!domainDoc) {
           throw new Error(
-            `DnsManager: no DNS provider configured for ${dnsChallenge.hostName}. ` +
-              'Add one in the Domains > Providers UI before issuing certificates.',
+            `DnsManager: no managed domain found for ${dnsChallenge.hostName}. ` +
+              'Add the domain in Domains before issuing certificates.',
           );
         }
-        // Clean any leftover challenge records first to avoid duplicates.
+        // Clean leftover challenge records first to avoid duplicates.
         try {
-          const existing = await client.listRecords(dnsChallenge.hostName);
-          for (const r of existing) {
-            if (r.type === 'TXT' && r.name === dnsChallenge.hostName) {
-              await client.deleteRecord(dnsChallenge.hostName, r.providerRecordId).catch(() => {});
-            }
-          }
+          await self.deleteRecordsByNameAndType(domainDoc.id, dnsChallenge.hostName, 'TXT');
         } catch (err: unknown) {
           logger.log('warn', `DnsManager: failed to clean existing TXT for ${dnsChallenge.hostName}: ${(err as Error).message}`);
         }
-        await client.createRecord(dnsChallenge.hostName, {
+        // Create the challenge TXT record via the unified path
+        await self.createRecord({
+          domainId: domainDoc.id,
           name: dnsChallenge.hostName,
           type: 'TXT',
           value: dnsChallenge.challenge,
           ttl: 120,
+          createdBy: 'acme-dns01',
         });
       },
       async acmeRemoveDnsChallenge(dnsChallenge: { hostName: string; challenge: string }) {
-        const client = await self.getProviderClientForDomain(dnsChallenge.hostName);
-        if (!client) {
+        const domainDoc = await self.findDomainForFqdn(dnsChallenge.hostName);
+        if (!domainDoc) {
           // The domain may have been removed; nothing to clean up.
           return;
         }
         try {
-          const existing = await client.listRecords(dnsChallenge.hostName);
-          for (const r of existing) {
-            if (r.type === 'TXT' && r.name === dnsChallenge.hostName) {
-              await client.deleteRecord(dnsChallenge.hostName, r.providerRecordId);
-            }
-          }
+          await self.deleteRecordsByNameAndType(domainDoc.id, dnsChallenge.hostName, 'TXT');
         } catch (err: unknown) {
           logger.log('warn', `DnsManager: failed to remove TXT for ${dnsChallenge.hostName}: ${(err as Error).message}`);
         }
       },
       async isDomainSupported(domain: string): Promise<boolean> {
-        const client = await self.getProviderClientForDomain(domain);
-        return !!client;
+        const domainDoc = await self.findDomainForFqdn(domain);
+        return !!domainDoc;
       },
     };
     return { convenience: adapter } as plugins.tsclass.network.IConvenientDnsProvider;
@@ -642,6 +671,151 @@ export class DnsManager {
     return await DnsRecordDoc.findById(id);
   }
 
+  // ==========================================================================
+  // Domain migration
+  // ==========================================================================
+
+  /**
+   * Migrate a domain between dcrouter-hosted and provider-managed.
+   * Transfers all records to the target and updates domain metadata.
+   */
+  public async migrateDomain(args: {
+    id: string;
+    targetSource: 'dcrouter' | 'provider';
+    targetProviderId?: string;
+    deleteExistingProviderRecords?: boolean;
+  }): Promise<{ success: boolean; recordsMigrated?: number; message?: string }> {
+    const domain = await DomainDoc.findById(args.id);
+    if (!domain) return { success: false, message: 'Domain not found' };
+
+    if (domain.source === args.targetSource && domain.providerId === args.targetProviderId) {
+      return { success: false, message: 'Domain is already in the target configuration' };
+    }
+
+    const records = await DnsRecordDoc.findByDomainId(domain.id);
+
+    if (args.targetSource === 'provider') {
+      return this.migrateToDnsProvider(domain, records, args.targetProviderId!, args.deleteExistingProviderRecords ?? false);
+    } else {
+      return this.migrateToDcrouter(domain, records);
+    }
+  }
+
+  /**
+   * Migrate domain from dcrouter-hosted (or another provider) to an external DNS provider.
+   */
+  private async migrateToDnsProvider(
+    domain: DomainDoc,
+    records: DnsRecordDoc[],
+    targetProviderId: string,
+    deleteExistingProviderRecords: boolean,
+  ): Promise<{ success: boolean; recordsMigrated?: number; message?: string }> {
+    // Validate the target provider exists
+    const client = await this.getProviderClientById(targetProviderId);
+    if (!client) {
+      return { success: false, message: 'Target DNS provider not found' };
+    }
+
+    // Find the zone at the provider
+    const providerDomains = await client.listDomains();
+    const zone = providerDomains.find(
+      (z) => z.name.toLowerCase() === domain.name.toLowerCase(),
+    );
+    if (!zone) {
+      return { success: false, message: `Zone "${domain.name}" not found at the target provider` };
+    }
+
+    // Optionally delete existing records at the provider
+    if (deleteExistingProviderRecords) {
+      try {
+        const existingProviderRecords = await client.listRecords(domain.name);
+        for (const pr of existingProviderRecords) {
+          await client.deleteRecord(domain.name, pr.providerRecordId).catch(() => {});
+        }
+        logger.log('info', `Deleted ${existingProviderRecords.length} existing records at provider for ${domain.name}`);
+      } catch (err: unknown) {
+        logger.log('warn', `Failed to clean existing provider records for ${domain.name}: ${(err as Error).message}`);
+      }
+    }
+
+    // Push each local record to the provider
+    let migrated = 0;
+    for (const rec of records) {
+      try {
+        const providerRecord = await client.createRecord(domain.name, {
+          name: rec.name,
+          type: rec.type as any,
+          value: rec.value,
+          ttl: rec.ttl,
+        });
+        // Unregister from embedded DnsServer if it was dcrouter-hosted
+        if (domain.source === 'dcrouter') {
+          this.unregisterRecordFromDnsServer(rec);
+        }
+        // Update the record doc to synced
+        rec.source = 'synced' as TDnsRecordSource;
+        rec.providerRecordId = providerRecord.providerRecordId;
+        await rec.save();
+        migrated++;
+      } catch (err: unknown) {
+        logger.log('warn', `Failed to migrate record ${rec.name} ${rec.type} to provider: ${(err as Error).message}`);
+      }
+    }
+
+    // Update domain metadata
+    domain.source = 'provider';
+    domain.authoritative = false;
+    domain.providerId = targetProviderId;
+    domain.externalZoneId = zone.externalId;
+    domain.nameservers = zone.nameservers;
+    domain.lastSyncedAt = Date.now();
+    domain.updatedAt = Date.now();
+    await domain.save();
+
+    logger.log('info', `Domain ${domain.name} migrated to provider (${migrated} records)`);
+    return { success: true, recordsMigrated: migrated };
+  }
+
+  /**
+   * Migrate domain from provider-managed to dcrouter-hosted (authoritative).
+   */
+  private async migrateToDcrouter(
+    domain: DomainDoc,
+    records: DnsRecordDoc[],
+  ): Promise<{ success: boolean; recordsMigrated?: number; message?: string }> {
+    // Register each record with the embedded DnsServer
+    let migrated = 0;
+    for (const rec of records) {
+      try {
+        this.registerRecordWithDnsServer(rec);
+        // Update the record doc to local
+        rec.source = 'local' as TDnsRecordSource;
+        rec.providerRecordId = undefined;
+        await rec.save();
+        migrated++;
+      } catch (err: unknown) {
+        logger.log('warn', `Failed to register record ${rec.name} ${rec.type} with DnsServer: ${(err as Error).message}`);
+      }
+    }
+
+    // Update domain metadata
+    domain.source = 'dcrouter';
+    domain.authoritative = true;
+    domain.providerId = undefined;
+    domain.externalZoneId = undefined;
+    domain.nameservers = undefined;
+    domain.lastSyncedAt = undefined;
+    domain.updatedAt = Date.now();
+    await domain.save();
+
+    logger.log('info', `Domain ${domain.name} migrated to dcrouter (${migrated} records)`);
+    return { success: true, recordsMigrated: migrated };
+  }
+
+  // ==========================================================================
+  // Record CRUD
+  // ==========================================================================
+
   public async createRecord(args: {
     domainId: string;
     name: string;
@@ -759,14 +933,24 @@ export class DnsManager {
         }
       }
     }
-    // For local records: smartdns has no unregister API in the pinned version,
-    // so the record stays served until the next restart. The DB delete still
-    // takes effect — on restart, the record will not be re-registered.
+    // For dcrouter-hosted records: unregister the handler from the embedded DnsServer
+    // so the record stops being served immediately (not just after restart).
+    if (domain.source === 'dcrouter' && this.dnsServer) {
+      this.unregisterRecordFromDnsServer(doc);
+    }
 
     await doc.delete();
     return { success: true };
   }
 
+  /**
+   * Unregister a record's handler from the embedded DnsServer.
+   */
+  public unregisterRecordFromDnsServer(rec: DnsRecordDoc): void {
+    if (!this.dnsServer) return;
+    this.dnsServer.unregisterHandler(rec.name, [rec.type]);
+  }
+
   // ==========================================================================
   // Internal helpers
   // ==========================================================================

package/ts/monitoring/classes.metricsmanager.ts

@@ -553,12 +553,14 @@ export class MetricsManager {
           connectionsByIP: new Map<string, number>(),
           throughputRate: { bytesInPerSecond: 0, bytesOutPerSecond: 0 },
           topIPs: [] as Array<{ ip: string; count: number }>,
+          topIPsByBandwidth: [] as Array<{ ip: string; count: number; bwIn: number; bwOut: number }>,
           totalDataTransferred: { bytesIn: 0, bytesOut: 0 },
           throughputHistory: [] as Array<{ timestamp: number; in: number; out: number }>,
           throughputByIP: new Map<string, { in: number; out: number }>(),
           requestsPerSecond: 0,
           requestsTotal: 0,
           backends: [] as Array<any>,
+          domainActivity: [] as Array<{ domain: string; bytesInPerSecond: number; bytesOutPerSecond: number; activeConnections: number; routeCount: number }>,
         };
       }
 
@@ -572,7 +574,7 @@ export class MetricsManager {
         bytesOutPerSecond: instantThroughput.out
       };
 
-      // Get top IPs
+      // Get top IPs by connection count
       const topIPs = proxyMetrics.connections.topIPs(10);
 
       // Get total data transferred
@@ -699,10 +701,83 @@ export class MetricsManager {
         }
       }
 
+      // Build top 10 IPs by bandwidth (sorted by total throughput desc)
+      const allIPData = new Map<string, { count: number; bwIn: number; bwOut: number }>();
+      for (const [ip, count] of connectionsByIP) {
+        allIPData.set(ip, { count, bwIn: 0, bwOut: 0 });
+      }
+      for (const [ip, tp] of throughputByIP) {
+        const existing = allIPData.get(ip);
+        if (existing) {
+          existing.bwIn = tp.in;
+          existing.bwOut = tp.out;
+        } else {
+          allIPData.set(ip, { count: 0, bwIn: tp.in, bwOut: tp.out });
+        }
+      }
+      const topIPsByBandwidth = Array.from(allIPData.entries())
+        .sort((a, b) => (b[1].bwIn + b[1].bwOut) - (a[1].bwIn + a[1].bwOut))
+        .slice(0, 10)
+        .map(([ip, data]) => ({ ip, count: data.count, bwIn: data.bwIn, bwOut: data.bwOut }));
+
+      // Build domain activity from per-route metrics
+      const connectionsByRoute = proxyMetrics.connections.byRoute();
+      const throughputByRoute = proxyMetrics.throughput.byRoute();
+
+      // Map route name → primary domain using dcrouter's route configs
+      const routeToDomain = new Map<string, string>();
+      if (this.dcRouter.smartProxy) {
+        for (const route of this.dcRouter.smartProxy.routeManager.getRoutes()) {
+          if (!route.name || !route.match.domains) continue;
+          const domains = Array.isArray(route.match.domains)
+            ? route.match.domains
+            : [route.match.domains];
+          if (domains.length > 0) {
+            routeToDomain.set(route.name, domains[0]);
+          }
+        }
+      }
+
+      // Aggregate metrics by domain
+      const domainAgg = new Map<string, {
+        activeConnections: number;
+        bytesInPerSec: number;
+        bytesOutPerSec: number;
+        routeCount: number;
+      }>();
+      for (const [routeName, activeConns] of connectionsByRoute) {
+        const domain = routeToDomain.get(routeName) || routeName;
+        const tp = throughputByRoute.get(routeName) || { in: 0, out: 0 };
+        const existing = domainAgg.get(domain);
+        if (existing) {
+          existing.activeConnections += activeConns;
+          existing.bytesInPerSec += tp.in;
+          existing.bytesOutPerSec += tp.out;
+          existing.routeCount++;
+        } else {
+          domainAgg.set(domain, {
+            activeConnections: activeConns,
+            bytesInPerSec: tp.in,
+            bytesOutPerSec: tp.out,
+            routeCount: 1,
+          });
+        }
+      }
+      const domainActivity = Array.from(domainAgg.entries())
+        .map(([domain, data]) => ({
+          domain,
+          bytesInPerSecond: data.bytesInPerSec,
+          bytesOutPerSecond: data.bytesOutPerSec,
+          activeConnections: data.activeConnections,
+          routeCount: data.routeCount,
+        }))
+        .sort((a, b) => (b.bytesInPerSecond + b.bytesOutPerSecond) - (a.bytesInPerSecond + a.bytesOutPerSecond));
+
       return {
         connectionsByIP,
         throughputRate,
         topIPs,
+        topIPsByBandwidth,
         totalDataTransferred,
         throughputHistory,
         throughputByIP,
@@ -711,6 +786,7 @@ export class MetricsManager {
         backends,
         frontendProtocols,
         backendProtocols,
+        domainActivity,
       };
     }, 1000); // 1s cache — matches typical dashboard poll interval
   }

package/ts/opsserver/handlers/config.handler.ts

@@ -127,7 +127,7 @@ export class ConfigHandler {
     // (replaces the legacy `dnsChallenge.cloudflareApiKey` constructor field).
     let dnsChallengeEnabled = false;
     try {
-      dnsChallengeEnabled = (await dcRouter.dnsManager?.hasAcmeCapableProvider()) ?? false;
+      dnsChallengeEnabled = (await dcRouter.dnsManager?.hasAnyManagedDomain()) ?? false;
     } catch {
       dnsChallengeEnabled = false;
     }

package/ts/opsserver/handlers/domain.handler.ts

@@ -157,5 +157,23 @@ export class DomainHandler {
         },
       ),
     );
+
+    // Migrate domain between dcrouter-hosted and provider-managed
+    this.typedrouter.addTypedHandler(
+      new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_MigrateDomain>(
+        'migrateDomain',
+        async (dataArg) => {
+          await this.requireAuth(dataArg, 'domains:write');
+          const dnsManager = this.opsServerRef.dcRouterRef.dnsManager;
+          if (!dnsManager) return { success: false, message: 'DnsManager not initialized' };
+          return await dnsManager.migrateDomain({
+            id: dataArg.id,
+            targetSource: dataArg.targetSource,
+            targetProviderId: dataArg.targetProviderId,
+            deleteExistingProviderRecords: dataArg.deleteExistingProviderRecords,
+          });
+        },
+      ),
+    );
   }
 }

package/ts/opsserver/handlers/security.handler.ts

@@ -51,8 +51,8 @@ export class SecurityHandler {
             startTime: conn.startTime,
             protocol: conn.type === 'http' ? 'https' : conn.type as any,
             state: conn.status as any,
-            bytesReceived: Math.floor(conn.bytesTransferred / 2),
-            bytesSent: Math.floor(conn.bytesTransferred / 2),
+            bytesReceived: (conn as any)._throughputIn || 0,
+            bytesSent: (conn as any)._throughputOut || 0,
           }));
           
           const summary = {
@@ -96,9 +96,11 @@ export class SecurityHandler {
               connectionsByIP: Array.from(networkStats.connectionsByIP.entries()).map(([ip, count]) => ({ ip, count })),
               throughputRate: networkStats.throughputRate,
               topIPs: networkStats.topIPs,
+              topIPsByBandwidth: networkStats.topIPsByBandwidth,
               totalDataTransferred: networkStats.totalDataTransferred,
               throughputHistory: networkStats.throughputHistory || [],
               throughputByIP,
+              domainActivity: networkStats.domainActivity || [],
               requestsPerSecond: networkStats.requestsPerSecond || 0,
               requestsTotal: networkStats.requestsTotal || 0,
               backends: networkStats.backends || [],
@@ -110,9 +112,11 @@ export class SecurityHandler {
             connectionsByIP: [],
             throughputRate: { bytesInPerSecond: 0, bytesOutPerSecond: 0 },
             topIPs: [],
+            topIPsByBandwidth: [],
             totalDataTransferred: { bytesIn: 0, bytesOut: 0 },
             throughputHistory: [],
             throughputByIP: [],
+            domainActivity: [],
             requestsPerSecond: 0,
             requestsTotal: 0,
             backends: [],
@@ -251,31 +255,31 @@ export class SecurityHandler {
       const connectionInfo = await this.opsServerRef.dcRouterRef.metricsManager.getConnectionInfo();
       const networkStats = await this.opsServerRef.dcRouterRef.metricsManager.getNetworkStats();
       
-      // Use IP-based connection data from the new metrics API
+      // One aggregate row per IP with real throughput data
       if (networkStats.connectionsByIP && networkStats.connectionsByIP.size > 0) {
         let connIndex = 0;
         const publicIp = this.opsServerRef.dcRouterRef.options.publicIp || 'server';
-        
+
         for (const [ip, count] of networkStats.connectionsByIP) {
-          // Create a connection entry for each active IP connection
-          for (let i = 0; i < Math.min(count, 5); i++) { // Limit to 5 connections per IP for UI performance
-            connections.push({
-              id: `conn-${connIndex++}`,
-              type: 'http',
-              source: {
-                ip: ip,
-                port: Math.floor(Math.random() * 50000) + 10000, // High port range
-              },
-              destination: {
-                ip: publicIp,
-                port: 443,
-                service: 'proxy',
-              },
-              startTime: Date.now() - Math.floor(Math.random() * 3600000), // Within last hour
-              bytesTransferred: Math.floor(networkStats.totalDataTransferred.bytesIn / networkStats.connectionsByIP.size),
-              status: 'active',
-            });
-          }
+          const tp = networkStats.throughputByIP?.get(ip);
+          connections.push({
+            id: `ip-${connIndex++}`,
+            type: 'http',
+            source: {
+              ip: ip,
+              port: 0,
+            },
+            destination: {
+              ip: publicIp,
+              port: 443,
+              service: 'proxy',
+            },
+            startTime: 0,
+            bytesTransferred: count, // Store connection count here
+            status: 'active',
+            // Attach real throughput for the handler mapping
+            ...(tp ? { _throughputIn: tp.in, _throughputOut: tp.out } : {}),
+          } as any);
         }
       } else if (connectionInfo.length > 0) {
         // Fallback to route-based connection info if no IP data available

package/ts/opsserver/handlers/stats.handler.ts

@@ -291,6 +291,20 @@ export class StatsHandler {
                   }
                 }
 
+                // Build connectionDetails from real per-IP data
+                const connectionDetails: interfaces.data.IConnectionDetails[] = [];
+                for (const [ip, count] of stats.connectionsByIP) {
+                  const tp = stats.throughputByIP?.get(ip);
+                  connectionDetails.push({
+                    remoteAddress: ip,
+                    protocol: 'https',
+                    state: 'connected',
+                    startTime: 0,
+                    bytesIn: tp?.in || 0,
+                    bytesOut: tp?.out || 0,
+                  });
+                }
+
                 metrics.network = {
                   totalBandwidth: {
                     in: stats.throughputRate.bytesInPerSecond,
@@ -301,12 +315,18 @@ export class StatsHandler {
                     out: stats.totalDataTransferred.bytesOut,
                   },
                   activeConnections: serverStats.activeConnections,
-                  connectionDetails: [],
+                  connectionDetails,
                   topEndpoints: stats.topIPs.map(ip => ({
                     endpoint: ip.ip,
-                    requests: ip.count,
+                    connections: ip.count,
                     bandwidth: ipBandwidth.get(ip.ip) || { in: 0, out: 0 },
                   })),
+                  topEndpointsByBandwidth: stats.topIPsByBandwidth.map(ip => ({
+                    endpoint: ip.ip,
+                    connections: ip.count,
+                    bandwidth: { in: ip.bwIn, out: ip.bwOut },
+                  })),
+                  domainActivity: stats.domainActivity || [],
                   throughputHistory: stats.throughputHistory || [],
                   requestsPerSecond: stats.requestsPerSecond || 0,
                   requestsTotal: stats.requestsTotal || 0,

package/ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '13.12.0',
+  version: '13.14.0',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }