@serve.zone/dcrouter 11.23.4 → 12.0.0

package/ts/classes.dcrouter.ts

@@ -11,12 +11,10 @@ import {
   type IEmailDomainConfig,
 } from '@push.rocks/smartmta';
 import { logger } from './logger.js';
-// Import storage manager
-import { StorageManager, type IStorageConfig } from './storage/index.js';
 import { StorageBackedCertManager } from './classes.storage-cert-manager.js';
 import { CertProvisionScheduler } from './classes.cert-provision-scheduler.js';
-// Import cache system
-import { CacheDb, CacheCleaner, type ICacheDbOptions } from './cache/index.js';
+// Import unified database
+import { DcRouterDb, type IDcRouterDbConfig, CacheCleaner, ProxyCertDoc, AcmeCertDoc } from './db/index.js';
 
 import { OpsServer } from './opsserver/index.js';
 import { MetricsManager } from './monitoring/index.js';
@@ -122,37 +120,23 @@ export interface IDcRouterOptions {
     /** Other DNS providers can be added here */
   };
   
-  /** Storage configuration */
-  storage?: IStorageConfig;
-
   /**
-   * Cache database configuration using smartdata and LocalTsmDb
-   * Provides persistent caching for emails, IP reputation, bounces, etc.
+   * Unified database configuration.
+   * All persistent data (config, certs, VPN, cache, etc.) is stored via smartdata.
+   * If mongoDbUrl is provided, connects to external MongoDB.
+   * Otherwise, starts an embedded LocalSmartDb automatically.
    */
-  cacheConfig?: {
-    /** Enable cache database (default: true) */
+  dbConfig?: {
+    /** Enable database (default: true). Set to false in tests to skip DB startup. */
     enabled?: boolean;
-    /** Storage path for TsmDB data (default: ~/.serve.zone/dcrouter/tsmdb) */
+    /** External MongoDB connection URL. If absent, uses embedded LocalSmartDb. */
+    mongoDbUrl?: string;
+    /** Storage path for embedded database data (default: ~/.serve.zone/dcrouter/tsmdb) */
     storagePath?: string;
     /** Database name (default: dcrouter) */
     dbName?: string;
-    /** Default TTL in days for cached items (default: 30) */
-    defaultTTLDays?: number;
-    /** Cleanup interval in hours (default: 1) */
+    /** Cache cleanup interval in hours (default: 1) */
     cleanupIntervalHours?: number;
-    /** TTL configuration per data type (in days) */
-    ttlConfig?: {
-      /** Email cache TTL (default: 30 days) */
-      emails?: number;
-      /** IP reputation cache TTL (default: 1 day) */
-      ipReputation?: number;
-      /** Bounce records TTL (default: 30 days) */
-      bounces?: number;
-      /** DKIM keys TTL (default: 90 days) */
-      dkimKeys?: number;
-      /** Suppression list TTL (default: 30 days, can be permanent) */
-      suppression?: number;
-    };
   };
 
   /**
@@ -248,12 +232,20 @@ export class DcRouter {
   public dnsServer?: plugins.smartdns.dnsServerMod.DnsServer;
   public emailServer?: UnifiedEmailServer;
   public radiusServer?: RadiusServer;
-  public storageManager: StorageManager;
   public opsServer!: OpsServer;
   public metricsManager?: MetricsManager;
 
-  // Cache system (smartdata + LocalTsmDb)
-  public cacheDb?: CacheDb;
+  // Compatibility shim for smartmta's DkimManager which calls dcRouter.storageManager.set()
+  public storageManager: any = {
+    get: async (_key: string) => null,
+    set: async (_key: string, _value: string) => {
+      // DKIM keys from smartmta — logged but not yet migrated to smartdata
+      logger.log('debug', `storageManager.set() called (compat shim) for key: ${_key}`);
+    },
+  };
+
+  // Unified database (smartdata + LocalSmartDb or external MongoDB)
+  public dcRouterDb?: DcRouterDb;
   public cacheCleaner?: CacheCleaner;
 
   // Remote Ingress
@@ -312,16 +304,6 @@ export class DcRouter {
     // Resolve all data paths from baseDir
     this.resolvedPaths = paths.resolvePaths(this.options.baseDir);
 
-    // Default storage to filesystem if not configured
-    if (!this.options.storage) {
-      this.options.storage = {
-        fsPath: this.resolvedPaths.defaultStoragePath,
-      };
-    }
-
-    // Initialize storage manager
-    this.storageManager = new StorageManager(this.options.storage);
-
     // Initialize service manager and register all services
     this.serviceManager = new plugins.taskbuffer.ServiceManager({
       name: 'dcrouter',
@@ -350,23 +332,23 @@ export class DcRouter {
         .withRetry({ maxRetries: 0 }),
     );
 
-    // CacheDb: optional, no dependencies
-    if (this.options.cacheConfig?.enabled !== false) {
+    // DcRouterDb: optional, no dependencies — unified database for all persistence
+    if (this.options.dbConfig?.enabled !== false) {
       this.serviceManager.addService(
-        new plugins.taskbuffer.Service('CacheDb')
+        new plugins.taskbuffer.Service('DcRouterDb')
           .optional()
           .withStart(async () => {
-            await this.setupCacheDb();
+            await this.setupDcRouterDb();
           })
           .withStop(async () => {
             if (this.cacheCleaner) {
               this.cacheCleaner.stop();
               this.cacheCleaner = undefined;
             }
-            if (this.cacheDb) {
-              await this.cacheDb.stop();
-              CacheDb.resetInstance();
-              this.cacheDb = undefined;
+            if (this.dcRouterDb) {
+              await this.dcRouterDb.stop();
+              DcRouterDb.resetInstance();
+              this.dcRouterDb = undefined;
             }
           })
           .withRetry({ maxRetries: 2, baseDelayMs: 1000, maxDelayMs: 5000 }),
@@ -391,10 +373,10 @@ export class DcRouter {
         .withRetry({ maxRetries: 1, baseDelayMs: 1000 }),
     );
 
-    // SmartProxy: critical, depends on CacheDb (if enabled)
+    // SmartProxy: critical, depends on DcRouterDb (if enabled)
     const smartProxyDeps: string[] = [];
-    if (this.options.cacheConfig?.enabled !== false) {
-      smartProxyDeps.push('CacheDb');
+    if (this.options.dbConfig?.enabled !== false) {
+      smartProxyDeps.push('DcRouterDb');
     }
     this.serviceManager.addService(
       new plugins.taskbuffer.Service('SmartProxy')
@@ -455,36 +437,38 @@ export class DcRouter {
       );
     }
 
-    // ConfigManagers: optional, depends on SmartProxy
-    this.serviceManager.addService(
-      new plugins.taskbuffer.Service('ConfigManagers')
-        .optional()
-        .dependsOn('SmartProxy')
-        .withStart(async () => {
-          this.routeConfigManager = new RouteConfigManager(
-            this.storageManager,
-            () => this.getConstructorRoutes(),
-            () => this.smartProxy,
-            () => this.options.http3,
-            this.options.vpnConfig?.enabled
-              ? (tags?: string[]) => {
-                  if (tags?.length && this.vpnManager) {
-                    return this.vpnManager.getClientIpsForServerDefinedTags(tags);
+    // ConfigManagers: optional, depends on SmartProxy + DcRouterDb
+    // Requires DcRouterDb to be enabled (document classes need the database)
+    if (this.options.dbConfig?.enabled !== false) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('ConfigManagers')
+          .optional()
+          .dependsOn('SmartProxy', 'DcRouterDb')
+          .withStart(async () => {
+            this.routeConfigManager = new RouteConfigManager(
+              () => this.getConstructorRoutes(),
+              () => this.smartProxy,
+              () => this.options.http3,
+              this.options.vpnConfig?.enabled
+                ? (tags?: string[]) => {
+                    if (tags?.length && this.vpnManager) {
+                      return this.vpnManager.getClientIpsForServerDefinedTags(tags);
+                    }
+                    return [this.options.vpnConfig?.subnet || '10.8.0.0/24'];
                   }
-                  return [this.options.vpnConfig?.subnet || '10.8.0.0/24'];
-                }
-              : undefined,
-          );
-          this.apiTokenManager = new ApiTokenManager(this.storageManager);
-          await this.apiTokenManager.initialize();
-          await this.routeConfigManager.initialize();
-        })
-        .withStop(async () => {
-          this.routeConfigManager = undefined;
-          this.apiTokenManager = undefined;
-        })
-        .withRetry({ maxRetries: 2, baseDelayMs: 1000 }),
-    );
+                : undefined,
+            );
+            this.apiTokenManager = new ApiTokenManager();
+            await this.apiTokenManager.initialize();
+            await this.routeConfigManager.initialize();
+          })
+          .withStop(async () => {
+            this.routeConfigManager = undefined;
+            this.apiTokenManager = undefined;
+          })
+          .withRetry({ maxRetries: 2, baseDelayMs: 1000 }),
+      );
+    }
 
     // Email Server: optional, depends on SmartProxy
     if (this.options.emailConfig) {
@@ -695,14 +679,9 @@ export class DcRouter {
       logger.log('info', `Remote Ingress: tunnel port=${this.options.remoteIngressConfig.tunnelPort || 8443}, edges=${edgeCount} registered/${connectedCount} connected`);
     }
 
-    // Storage summary
-    if (this.storageManager && this.options.storage) {
-      logger.log('info', `Storage: path=${this.options.storage.fsPath || 'default'}`);
-    }
-
-    // Cache database summary
-    if (this.cacheDb) {
-      logger.log('info', `Cache Database: storage=${this.cacheDb.getStoragePath()}, db=${this.cacheDb.getDbName()}, cleaner=${this.cacheCleaner?.isActive() ? 'active' : 'inactive'} (${(this.options.cacheConfig?.cleanupIntervalHours || 1)}h interval)`);
+    // Database summary
+    if (this.dcRouterDb) {
+      logger.log('info', `Database: ${this.dcRouterDb.isEmbedded() ? 'embedded' : 'external'}, db=${this.dcRouterDb.getDbName()}, cleaner=${this.cacheCleaner?.isActive() ? 'active' : 'inactive'} (${(this.options.dbConfig?.cleanupIntervalHours || 1)}h interval)`);
     }
 
     // Service status summary from ServiceManager
@@ -723,31 +702,32 @@ export class DcRouter {
   }
   
   /**
-   * Set up the cache database (smartdata + LocalTsmDb)
+   * Set up the unified database (smartdata + LocalSmartDb or external MongoDB)
    */
-  private async setupCacheDb(): Promise<void> {
-    logger.log('info', 'Setting up CacheDb...');
+  private async setupDcRouterDb(): Promise<void> {
+    logger.log('info', 'Setting up DcRouterDb...');
 
-    const cacheConfig = this.options.cacheConfig || {};
+    const dbConfig = this.options.dbConfig || {};
 
-    // Initialize CacheDb singleton
-    this.cacheDb = CacheDb.getInstance({
-      storagePath: cacheConfig.storagePath || this.resolvedPaths.defaultTsmDbPath,
-      dbName: cacheConfig.dbName || 'dcrouter',
+    // Initialize DcRouterDb singleton
+    this.dcRouterDb = DcRouterDb.getInstance({
+      mongoDbUrl: dbConfig.mongoDbUrl,
+      storagePath: dbConfig.storagePath || this.resolvedPaths.defaultTsmDbPath,
+      dbName: dbConfig.dbName || 'dcrouter',
       debug: false,
     });
 
-    await this.cacheDb.start();
+    await this.dcRouterDb.start();
 
-    // Start the cache cleaner
-    const cleanupIntervalMs = (cacheConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
-    this.cacheCleaner = new CacheCleaner(this.cacheDb, {
+    // Start the cache cleaner for TTL-based document cleanup
+    const cleanupIntervalMs = (dbConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
+    this.cacheCleaner = new CacheCleaner(this.dcRouterDb, {
       intervalMs: cleanupIntervalMs,
       verbose: false,
     });
     this.cacheCleaner.start();
 
-    logger.log('info', `CacheDb initialized at ${this.cacheDb.getStoragePath()}`);
+    logger.log('info', `DcRouterDb ready (${this.dcRouterDb.isEmbedded() ? 'embedded' : 'external'})`);
   }
 
   /**
@@ -850,14 +830,11 @@ export class DcRouter {
         acme: acmeConfig,
         certStore: {
           loadAll: async () => {
-            const keys = await this.storageManager.list('/proxy-certs/');
+            const docs = await ProxyCertDoc.findAll();
             const certs: Array<{ domain: string; publicKey: string; privateKey: string; ca?: string }> = [];
-            for (const key of keys) {
-              const data = await this.storageManager.getJSON(key);
-              if (data) {
-                certs.push(data);
-                loadedCertEntries.push({ domain: data.domain, publicKey: data.publicKey, validUntil: data.validUntil, validFrom: data.validFrom });
-              }
+            for (const doc of docs) {
+              certs.push({ domain: doc.domain, publicKey: doc.publicKey, privateKey: doc.privateKey, ca: doc.ca });
+              loadedCertEntries.push({ domain: doc.domain, publicKey: doc.publicKey, validUntil: doc.validUntil, validFrom: doc.validFrom });
             }
             return certs;
           },
@@ -869,18 +846,29 @@ export class DcRouter {
               validUntil = new Date(x509.validTo).getTime();
               validFrom = new Date(x509.validFrom).getTime();
             } catch { /* PEM parsing failed */ }
-            await this.storageManager.setJSON(`/proxy-certs/${domain}`, {
-              domain, publicKey, privateKey, ca, validUntil, validFrom,
-            });
+            let doc = await ProxyCertDoc.findByDomain(domain);
+            if (!doc) {
+              doc = new ProxyCertDoc();
+              doc.domain = domain;
+            }
+            doc.publicKey = publicKey;
+            doc.privateKey = privateKey;
+            doc.ca = ca || '';
+            doc.validUntil = validUntil || 0;
+            doc.validFrom = validFrom || 0;
+            await doc.save();
           },
           remove: async (domain: string) => {
-            await this.storageManager.delete(`/proxy-certs/${domain}`);
+            const doc = await ProxyCertDoc.findByDomain(domain);
+            if (doc) {
+              await doc.delete();
+            }
           },
         },
       };
       
       // Initialize cert provision scheduler
-      this.certProvisionScheduler = new CertProvisionScheduler(this.storageManager);
+      this.certProvisionScheduler = new CertProvisionScheduler();
 
       // If we have DNS challenge handlers, create SmartAcme instance and wire certProvisionFunction
       // Note: SmartAcme.start() is NOT called here — it runs as a separate optional service
@@ -895,7 +883,7 @@ export class DcRouter {
         }
         this.smartAcme = new plugins.smartacme.SmartAcme({
           accountEmail: acmeConfig?.accountEmail || this.options.tls?.contactEmail || 'admin@example.com',
-          certManager: new StorageBackedCertManager(this.storageManager),
+          certManager: new StorageBackedCertManager(),
           environment: 'production',
           challengeHandlers: challengeHandlers,
           challengePriority: ['dns-01'],
@@ -1037,16 +1025,16 @@ export class DcRouter {
             issuedAt = new Date(entry.validFrom).toISOString();
           }
 
-          // Try SmartAcme /certs/ metadata as secondary source
+          // Try SmartAcme AcmeCertDoc metadata as secondary source
           if (!expiryDate) {
             try {
               const cleanDomain = entry.domain.replace(/^\*\.?/, '');
-              const certMeta = await this.storageManager.getJSON(`/certs/${cleanDomain}`);
-              if (certMeta?.validUntil) {
-                expiryDate = new Date(certMeta.validUntil).toISOString();
+              const certDoc = await AcmeCertDoc.findByDomain(cleanDomain);
+              if (certDoc?.validUntil) {
+                expiryDate = new Date(certDoc.validUntil).toISOString();
               }
-              if (certMeta?.created && !issuedAt) {
-                issuedAt = new Date(certMeta.created).toISOString();
+              if (certDoc?.created && !issuedAt) {
+                issuedAt = new Date(certDoc.created).toISOString();
               }
             } catch { /* no metadata available */ }
           }
@@ -2030,7 +2018,7 @@ export class DcRouter {
     logger.log('info', 'Setting up Remote Ingress hub...');
 
     // Initialize the edge registration manager
-    this.remoteIngressManager = new RemoteIngressManager(this.storageManager);
+    this.remoteIngressManager = new RemoteIngressManager();
     await this.remoteIngressManager.initialize();
 
     // Pass current routes so the manager can derive edge ports from remoteIngress-tagged routes
@@ -2056,7 +2044,7 @@ export class DcRouter {
     // Priority 2: Existing cert from SmartProxy cert store for hubDomain
     if (!tlsConfig && riCfg.hubDomain) {
       try {
-        const stored = await this.storageManager.getJSON(`/proxy-certs/${riCfg.hubDomain}`);
+        const stored = await ProxyCertDoc.findByDomain(riCfg.hubDomain);
         if (stored?.publicKey && stored?.privateKey) {
           tlsConfig = { certPem: stored.publicKey, keyPem: stored.privateKey };
           logger.log('info', `Using stored ACME cert for RemoteIngress tunnel TLS: ${riCfg.hubDomain}`);
@@ -2090,7 +2078,7 @@ export class DcRouter {
 
     logger.log('info', 'Setting up VPN server...');
 
-    this.vpnManager = new VpnManager(this.storageManager, {
+    this.vpnManager = new VpnManager({
       subnet: this.options.vpnConfig.subnet,
       wgListenPort: this.options.vpnConfig.wgListenPort,
       dns: this.options.vpnConfig.dns,
@@ -2180,7 +2168,7 @@ export class DcRouter {
 
     logger.log('info', 'Setting up RADIUS server...');
 
-    this.radiusServer = new RadiusServer(this.options.radiusConfig, this.storageManager);
+    this.radiusServer = new RadiusServer(this.options.radiusConfig);
     await this.radiusServer.start();
 
     logger.log('info', `RADIUS server started on ports ${this.options.radiusConfig.authPort || 1812} (auth) and ${this.options.radiusConfig.acctPort || 1813} (acct)`);

package/ts/classes.storage-cert-manager.ts

@@ -1,46 +1,58 @@
 import * as plugins from './plugins.js';
-import { StorageManager } from './storage/index.js';
+import { AcmeCertDoc } from './db/index.js';
 
 /**
- * ICertManager implementation backed by StorageManager.
- * Persists SmartAcme certificates under a /certs/ key prefix so they
+ * ICertManager implementation backed by smartdata document classes.
+ * Persists SmartAcme certificates via AcmeCertDoc so they
  * survive process restarts without re-hitting ACME.
  */
 export class StorageBackedCertManager implements plugins.smartacme.ICertManager {
-  private keyPrefix = '/certs/';
-
-  constructor(private storageManager: StorageManager) {}
+  constructor() {}
 
   async init(): Promise<void> {}
 
   async retrieveCertificate(domainName: string): Promise<plugins.smartacme.Cert | null> {
-    const data = await this.storageManager.getJSON(this.keyPrefix + domainName);
-    if (!data) return null;
-    return new plugins.smartacme.Cert(data);
+    const doc = await AcmeCertDoc.findByDomain(domainName);
+    if (!doc) return null;
+    return new plugins.smartacme.Cert({
+      id: doc.id,
+      domainName: doc.domainName,
+      created: doc.created,
+      privateKey: doc.privateKey,
+      publicKey: doc.publicKey,
+      csr: doc.csr,
+      validUntil: doc.validUntil,
+    });
   }
 
   async storeCertificate(cert: plugins.smartacme.Cert): Promise<void> {
-    await this.storageManager.setJSON(this.keyPrefix + cert.domainName, {
-      id: cert.id,
-      domainName: cert.domainName,
-      created: cert.created,
-      privateKey: cert.privateKey,
-      publicKey: cert.publicKey,
-      csr: cert.csr,
-      validUntil: cert.validUntil,
-    });
+    let doc = await AcmeCertDoc.findByDomain(cert.domainName);
+    if (!doc) {
+      doc = new AcmeCertDoc();
+      doc.domainName = cert.domainName;
+    }
+    doc.id = cert.id;
+    doc.created = cert.created;
+    doc.privateKey = cert.privateKey;
+    doc.publicKey = cert.publicKey;
+    doc.csr = cert.csr;
+    doc.validUntil = cert.validUntil;
+    await doc.save();
   }
 
   async deleteCertificate(domainName: string): Promise<void> {
-    await this.storageManager.delete(this.keyPrefix + domainName);
+    const doc = await AcmeCertDoc.findByDomain(domainName);
+    if (doc) {
+      await doc.delete();
+    }
   }
 
   async close(): Promise<void> {}
 
   async wipe(): Promise<void> {
-    const keys = await this.storageManager.list(this.keyPrefix);
-    for (const key of keys) {
-      await this.storageManager.delete(key);
+    const docs = await AcmeCertDoc.findAll();
+    for (const doc of docs) {
+      await doc.delete();
     }
   }
 }

package/ts/config/classes.api-token-manager.ts

@@ -1,19 +1,18 @@
 import * as plugins from '../plugins.js';
 import { logger } from '../logger.js';
-import type { StorageManager } from '../storage/index.js';
+import { ApiTokenDoc } from '../db/index.js';
 import type {
   IStoredApiToken,
   IApiTokenInfo,
   TApiTokenScope,
 } from '../../ts_interfaces/data/route-management.js';
 
-const TOKENS_PREFIX = '/config-api/tokens/';
 const TOKEN_PREFIX_STR = 'dcr_';
 
 export class ApiTokenManager {
   private tokens = new Map<string, IStoredApiToken>();
 
-  constructor(private storageManager: StorageManager) {}
+  constructor() {}
 
   public async initialize(): Promise<void> {
     await this.loadTokens();
@@ -117,7 +116,8 @@ export class ApiTokenManager {
     if (!this.tokens.has(id)) return false;
     const token = this.tokens.get(id)!;
     this.tokens.delete(id);
-    await this.storageManager.delete(`${TOKENS_PREFIX}${id}.json`);
+    const doc = await ApiTokenDoc.findById(id);
+    if (doc) await doc.delete();
     logger.log('info', `API token '${token.name}' revoked (id: ${id})`);
     return true;
   }
@@ -157,17 +157,48 @@ export class ApiTokenManager {
   // =========================================================================
 
   private async loadTokens(): Promise<void> {
-    const keys = await this.storageManager.list(TOKENS_PREFIX);
-    for (const key of keys) {
-      if (!key.endsWith('.json')) continue;
-      const stored = await this.storageManager.getJSON<IStoredApiToken>(key);
-      if (stored?.id) {
-        this.tokens.set(stored.id, stored);
+    const docs = await ApiTokenDoc.findAll();
+    for (const doc of docs) {
+      if (doc.id) {
+        this.tokens.set(doc.id, {
+          id: doc.id,
+          name: doc.name,
+          tokenHash: doc.tokenHash,
+          scopes: doc.scopes,
+          createdAt: doc.createdAt,
+          expiresAt: doc.expiresAt,
+          lastUsedAt: doc.lastUsedAt,
+          createdBy: doc.createdBy,
+          enabled: doc.enabled,
+        });
       }
     }
   }
 
   private async persistToken(stored: IStoredApiToken): Promise<void> {
-    await this.storageManager.setJSON(`${TOKENS_PREFIX}${stored.id}.json`, stored);
+    const existing = await ApiTokenDoc.findById(stored.id);
+    if (existing) {
+      existing.name = stored.name;
+      existing.tokenHash = stored.tokenHash;
+      existing.scopes = stored.scopes;
+      existing.createdAt = stored.createdAt;
+      existing.expiresAt = stored.expiresAt;
+      existing.lastUsedAt = stored.lastUsedAt;
+      existing.createdBy = stored.createdBy;
+      existing.enabled = stored.enabled;
+      await existing.save();
+    } else {
+      const doc = new ApiTokenDoc();
+      doc.id = stored.id;
+      doc.name = stored.name;
+      doc.tokenHash = stored.tokenHash;
+      doc.scopes = stored.scopes;
+      doc.createdAt = stored.createdAt;
+      doc.expiresAt = stored.expiresAt;
+      doc.lastUsedAt = stored.lastUsedAt;
+      doc.createdBy = stored.createdBy;
+      doc.enabled = stored.enabled;
+      await doc.save();
+    }
   }
 }