@serve.zone/dcrouter 15.11.0 → 16.0.0

package/ts/opsserver/handlers/email-domain.handler.ts

@@ -73,6 +73,7 @@ export class EmailDomainHandler {
               dkimKeySize: dataArg.dkimKeySize,
               rotateKeys: dataArg.rotateKeys,
               rotationIntervalDays: dataArg.rotationIntervalDays,
+              remoteIngress: dataArg.remoteIngress,
             });
             return { success: true, domain };
           } catch (err: unknown) {
@@ -96,6 +97,7 @@ export class EmailDomainHandler {
               rotateKeys: dataArg.rotateKeys,
               rotationIntervalDays: dataArg.rotationIntervalDays,
               rateLimits: dataArg.rateLimits,
+              remoteIngress: dataArg.remoteIngress,
             });
             return { success: true };
           } catch (err: unknown) {

package/ts/opsserver/handlers/email-ops.handler.ts

@@ -2,6 +2,7 @@ import * as plugins from '../../plugins.js';
 import type { OpsServer } from '../classes.opsserver.js';
 import * as interfaces from '../../../ts_interfaces/index.js';
 import { requireOpsAuth } from '../helpers/auth.js';
+import { CachedEmail } from '../../db/index.js';
 
 export class EmailOpsHandler {
   constructor(private opsServerRef: OpsServer) {
@@ -20,7 +21,7 @@ export class EmailOpsHandler {
         'getAllEmails',
         async (dataArg) => {
           await requireOpsAuth(this.opsServerRef, dataArg, { scope: 'emails:read' });
-          const emails = this.getAllQueueEmails();
+          const emails = await this.getAllEmails(dataArg.direction, dataArg.limit);
           return { emails };
         }
       )
@@ -32,7 +33,7 @@ export class EmailOpsHandler {
         'getEmailDetail',
         async (dataArg) => {
           await requireOpsAuth(this.opsServerRef, dataArg, { scope: 'emails:read' });
-          const email = this.getEmailDetail(dataArg.emailId);
+          const email = await this.getEmailDetail(dataArg.emailId);
           return { email };
         }
       )
@@ -85,36 +86,172 @@ export class EmailOpsHandler {
   }
 
   /**
-   * Get all queue items mapped to catalog IEmail format
+   * List accepted emails (inbound + outbound) from the durable CachedEmail
+   * store, plus any live outbound queue items not yet represented there.
    */
-  private getAllQueueEmails(): interfaces.requests.IEmail[] {
-    const emailServer = this.opsServerRef.dcRouterRef.emailServer;
-    if (!emailServer) {
-      return [];
+  private async getAllEmails(
+    direction?: interfaces.requests.TEmailDirection,
+    limitArg?: number,
+  ): Promise<interfaces.requests.IEmail[]> {
+    const limit = Math.min(Math.max(Math.floor(limitArg ?? 200), 1), 1000);
+    const cachedDocs = this.isCachedEmailStoreAvailable()
+      ? await CachedEmail.findRecent(limit, { direction })
+      : [];
+    const emails = cachedDocs.map((doc) => this.mapCachedEmailToEmail(doc));
+    const representedIds = new Set(cachedDocs.map((doc) => doc.id));
+
+    // Live delivery-queue items are always outbound; every spooled message
+    // carries the cache-id header, so dedupe against the cached listing.
+    if (direction !== 'inbound') {
+      const emailServer = this.opsServerRef.dcRouterRef.emailServer;
+      const spool = this.opsServerRef.dcRouterRef.acceptedEmailSpool;
+      for (const item of emailServer?.getQueueItems() ?? []) {
+        const cachedId = spool.getCachedEmailIdFromQueueItem(item);
+        if (cachedId && representedIds.has(cachedId)) {
+          continue;
+        }
+        emails.push(this.mapQueueItemToEmail(item));
+      }
     }
-    const emails = emailServer.getQueueItems().map((item) => this.mapQueueItemToEmail(item));
 
-    // Sort by createdAt descending (newest first)
+    // Sort by timestamp descending (newest first)
     emails.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
 
-    return emails;
+    return emails.slice(0, limit);
   }
 
   /**
-   * Get a single email detail by ID
+   * Get a single email detail by ID — live queue item first (freshest state),
+   * else the durable CachedEmail row (covers inbound + historical mail).
    */
-  private getEmailDetail(emailId: string): interfaces.requests.IEmailDetail | null {
+  private async getEmailDetail(emailId: string): Promise<interfaces.requests.IEmailDetail | null> {
     const emailServer = this.opsServerRef.dcRouterRef.emailServer;
-    if (!emailServer) {
-      return null;
+    const item = emailServer?.getQueueItem(emailId);
+    if (item) {
+      return this.mapQueueItemToEmailDetail(item);
+    }
+
+    if (this.isCachedEmailStoreAvailable()) {
+      const cachedEmail = await CachedEmail.findById(emailId);
+      if (cachedEmail) {
+        return this.mapCachedEmailToEmailDetail(cachedEmail);
+      }
+    }
+    return null;
+  }
+
+  /** Durable listing needs the DB; without it only the live queue is visible. */
+  private isCachedEmailStoreAvailable(): boolean {
+    return this.opsServerRef.dcRouterRef.dcRouterDb?.isReady() === true;
+  }
+
+  /**
+   * Map a CachedEmail row to catalog IEmail format
+   */
+  private mapCachedEmailToEmail(doc: CachedEmail): interfaces.requests.IEmail {
+    return {
+      id: doc.id,
+      direction: doc.direction,
+      status: this.mapStatus(doc.status),
+      from: doc.from || '',
+      to: doc.to?.[0] || '',
+      subject: doc.subject || '',
+      timestamp: new Date(doc.acceptedAt || 0).toISOString(),
+      messageId: doc.messageId || '',
+      size: this.formatSize(doc.rawContent?.length || 0),
+    };
+  }
+
+  /**
+   * Map a CachedEmail row to catalog IEmailDetail format
+   */
+  private mapCachedEmailToEmailDetail(doc: CachedEmail): interfaces.requests.IEmailDetail {
+    const base = this.mapCachedEmailToEmail(doc);
+    const { headers, body } = this.parseRawEmail(doc.rawContent || '');
+    const session = this.parseCachedEmailSession(doc);
+
+    return {
+      ...base,
+      toList: doc.to || [],
+      cc: doc.cc || [],
+      smtpLog: [],
+      connectionInfo: {
+        sourceIp: session.remoteAddress || '',
+        sourceHostname: session.clientHostname || '',
+        destinationIp: '',
+        destinationPort: 0,
+        tlsVersion: session.secure ? 'TLS' : '',
+        tlsCipher: '',
+        authenticated: !!session.authenticated,
+        authMethod: '',
+        authUser: this.extractSessionUser(session),
+      },
+      authenticationResults: {
+        spf: 'none',
+        spfDomain: '',
+        dkim: 'none',
+        dkimDomain: '',
+        dmarc: 'none',
+        dmarcPolicy: '',
+      },
+      rejectionReason: doc.status === 'failed' ? doc.lastError : undefined,
+      bounceMessage: doc.status === 'failed' ? doc.lastError : undefined,
+      headers,
+      body,
+    };
+  }
+
+  /** Session snapshot persisted in routeData at acceptance time. */
+  private parseCachedEmailSession(doc: CachedEmail): {
+    remoteAddress?: string;
+    clientHostname?: string;
+    secure?: boolean;
+    authenticated?: boolean;
+    user?: unknown;
+  } {
+    try {
+      const routeData = doc.routeData ? JSON.parse(doc.routeData) : {};
+      return routeData.session || {};
+    } catch {
+      return {};
     }
-    const item = emailServer.getQueueItem(emailId);
+  }
 
-    if (!item) {
-      return null;
+  private extractSessionUser(session: { user?: unknown }): string {
+    if (!session.user) {
+      return '';
     }
+    if (typeof session.user === 'string') {
+      return session.user;
+    }
+    const user = session.user as { username?: string; id?: string };
+    return user.username || user.id || '';
+  }
+
+  /** Split raw RFC822 content into an unfolded header map and the body. */
+  private parseRawEmail(rawContent: string): { headers: Record<string, string>; body: string } {
+    const separatorMatch = rawContent.match(/\r?\n\r?\n/);
+    const headerBlock = separatorMatch
+      ? rawContent.slice(0, separatorMatch.index)
+      : rawContent;
+    const body = separatorMatch
+      ? rawContent.slice((separatorMatch.index || 0) + separatorMatch[0].length)
+      : '';
 
-    return this.mapQueueItemToEmailDetail(item);
+    const headers: Record<string, string> = {};
+    let currentKey: string | undefined;
+    for (const line of headerBlock.split(/\r?\n/)) {
+      if (/^[\t ]/.test(line) && currentKey) {
+        headers[currentKey] += ` ${line.trim()}`;
+        continue;
+      }
+      const idx = line.indexOf(':');
+      if (idx > 0) {
+        currentKey = line.slice(0, idx).trim();
+        headers[currentKey] = line.slice(idx + 1).trim();
+      }
+    }
+    return { headers, body };
   }
 
   /**
@@ -243,6 +380,7 @@ export class EmailOpsHandler {
     switch (queueStatus) {
       case 'pending':
       case 'processing':
+      case 'queued':
         return 'pending';
       case 'delivered':
         return 'delivered';

package/ts/opsserver/handlers/email-settings.handler.ts

@@ -49,16 +49,21 @@ export class EmailSettingsHandler {
   }
 
   private getSettings(): interfaces.data.IEmailServerSettings {
-    const manager = this.opsServerRef.dcRouterRef.emailSettingsManager;
+    const dcRouter = this.opsServerRef.dcRouterRef;
+    const outboundEgress = dcRouter.getOutboundEgressStatus();
+    const manager = dcRouter.emailSettingsManager;
     if (manager) {
-      return manager.getPublicSettings();
+      return { ...manager.getPublicSettings(), outboundEgress };
     }
-    const emailConfig = this.opsServerRef.dcRouterRef.options.emailConfig;
-    const emailPortConfig = this.opsServerRef.dcRouterRef.options.emailPortConfig;
+    const emailConfig = dcRouter.options.emailConfig;
+    const emailPortConfig = dcRouter.options.emailPortConfig;
     return {
       enabled: Boolean(emailConfig),
       hostname: emailConfig?.hostname || null,
-      outboundMode: this.opsServerRef.dcRouterRef.options.emailOutboundMode || 'direct',
+      // No settings manager yet: report the config seed; absent means the
+      // 'remoteIngress' default applies — never silently claim 'direct'.
+      outboundMode: dcRouter.options.emailOutboundMode || 'remoteIngress',
+      outboundEgress,
       ports: [...(emailConfig?.ports || [])],
       portMapping: emailPortConfig?.portMapping ? { ...emailPortConfig.portMapping } : null,
       receivedEmailsPath: emailPortConfig?.receivedEmailsPath || null,

package/ts/opsserver/handlers/remoteingress.handler.ts

@@ -64,12 +64,18 @@ export class RemoteIngressHandler {
               dataArg.autoDerivePorts ?? true,
               dataArg.performance,
               dataArg.egress,
+              {
+                publicIp: dataArg.publicIp,
+                publicIpV6: dataArg.publicIpV6,
+                mailHostname: dataArg.mailHostname,
+              },
             ));
             return { success: true, edge };
           } catch (err: unknown) {
             return {
               success: false,
               edge: null as any,
+              message: (err as Error).message,
             };
           }
         },
@@ -111,6 +117,7 @@ export class RemoteIngressHandler {
             scope: 'remote-ingress:write',
             requireAdminIdentity: true,
           });
+          let failureMessage: string | undefined;
           const result = await this.opsServerRef.dcRouterRef.mutateRemoteIngressEdges(async (manager) => {
             const edge = await manager.updateEdge(dataArg.id, {
               name: dataArg.name,
@@ -120,6 +127,9 @@ export class RemoteIngressHandler {
               performance: dataArg.performance,
               egress: dataArg.egress,
               tags: dataArg.tags,
+              publicIp: dataArg.publicIp,
+              publicIpV6: dataArg.publicIpV6,
+              mailHostname: dataArg.mailHostname,
             });
 
             if (!edge) {
@@ -136,14 +146,14 @@ export class RemoteIngressHandler {
               derivedPorts: breakdown.derived,
             };
           }).catch((err: unknown) => {
-            if ((err as Error).message.includes('RemoteIngress')) {
-              return null;
-            }
-            throw err;
+            // Validation rejections (bad IP, hub-IP guard, bad FQDN) and a
+            // missing manager both surface as success:false with the reason.
+            failureMessage = (err as Error).message;
+            return null;
           });
 
           if (!result) {
-            return { success: false, edge: null as any };
+            return { success: false, edge: null as any, message: failureMessage };
           }
           return {
             success: true,

package/ts/remoteingress/classes.remoteingress-manager.ts

@@ -1,7 +1,17 @@
 import * as plugins from '../plugins.js';
 import type { IDcRouterRouteConfig, IRemoteIngress, IRemoteIngressEgressPolicyConfig, IRemoteIngressHubSettings, IRemoteIngressPerformanceConfig, TRemoteIngressHubSettingsUpdate, TRemoteIngressPerformanceProfile } from '../../ts_interfaces/data/remoteingress.js';
+import { REMOTE_INGRESS_MAIL_TAG } from '../../ts_interfaces/data/remoteingress.js';
 import { RemoteIngressEdgeDoc, RemoteIngressHubSettingsDoc } from '../db/index.js';
 
+/** Operator-declared host identity of an edge (public IPs + PTR-aligned mail hostname). */
+export interface IRemoteIngressEdgeHostInfo {
+  publicIp?: string;
+  publicIpV6?: string;
+  mailHostname?: string;
+}
+
+const fqdnRegex = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/i;
+
 export interface IRemoteIngressFirewallConfig {
   blockedIps?: string[];
 }
@@ -51,6 +61,8 @@ export class RemoteIngressManager {
   private edges: Map<string, IRemoteIngress> = new Map();
   private routes: IDcRouterRouteConfig[] = [];
   private firewallConfig?: IRemoteIngressFirewallConfig;
+  private hubPublicIps: string[] = [];
+  private onEdgesChanged?: (reason: string) => void;
   private hubSettings: IRemoteIngressHubSettings = {
     enabled: false,
     tunnelPort: defaultTunnelPort,
@@ -60,6 +72,26 @@ export class RemoteIngressManager {
 
   constructor() {}
 
+  /**
+   * Set the hub's own public IPs. Edge publicIp values equal to any of these
+   * are rejected — the hub IP must never become an edge-fronted DNS target.
+   */
+  public setHubPublicIps(ips: Array<string | undefined>): void {
+    this.hubPublicIps = ips.filter((ip): ip is string => Boolean(ip));
+  }
+
+  /**
+   * Register a callback fired after every edge mutation (create/update/delete)
+   * so dependent state — mail DNS records — can reconcile.
+   */
+  public setOnEdgesChanged(callback: (reason: string) => void): void {
+    this.onEdgesChanged = callback;
+  }
+
+  private notifyEdgesChanged(reason: string): void {
+    this.onEdgesChanged?.(reason);
+  }
+
   /**
    * Load all edge registrations from the database into memory.
    */
@@ -82,6 +114,9 @@ export class RemoteIngressManager {
         performance: doc.performance,
         egress: this.normalizeEgressPolicy(doc.egress),
         tags: doc.tags,
+        publicIp: doc.publicIp,
+        publicIpV6: doc.publicIpV6,
+        mailHostname: doc.mailHostname,
         createdAt: doc.createdAt,
         updatedAt: doc.updatedAt,
       };
@@ -277,6 +312,7 @@ export class RemoteIngressManager {
     autoDerivePorts: boolean = true,
     performance?: IRemoteIngressPerformanceConfig,
     egress?: IRemoteIngressEgressPolicyConfig,
+    hostInfo?: IRemoteIngressEdgeHostInfo,
   ): Promise<IRemoteIngress> {
     const id = plugins.uuid.v4();
     const secret = plugins.crypto.randomBytes(32).toString('hex');
@@ -292,6 +328,9 @@ export class RemoteIngressManager {
       performance,
       egress: this.normalizeEgressPolicy(egress),
       tags: tags || [],
+      publicIp: this.normalizeEdgePublicIp(hostInfo?.publicIp, 4),
+      publicIpV6: this.normalizeEdgePublicIp(hostInfo?.publicIpV6, 6),
+      mailHostname: this.normalizeMailHostname(hostInfo?.mailHostname),
       createdAt: now,
       updatedAt: now,
     };
@@ -300,6 +339,7 @@ export class RemoteIngressManager {
     Object.assign(doc, edge);
     await doc.save();
     this.edges.set(id, edge);
+    this.notifyEdgesChanged(`edge created: ${name}`);
     return edge;
   }
 
@@ -330,6 +370,10 @@ export class RemoteIngressManager {
       performance?: IRemoteIngressPerformanceConfig;
       egress?: IRemoteIngressEgressPolicyConfig;
       tags?: string[];
+      /** null clears the stored value. */
+      publicIp?: string | null;
+      publicIpV6?: string | null;
+      mailHostname?: string | null;
     },
   ): Promise<IRemoteIngress | null> {
     const edge = this.edges.get(id);
@@ -344,6 +388,15 @@ export class RemoteIngressManager {
     if (updates.performance !== undefined) edge.performance = updates.performance;
     if (updates.egress !== undefined) edge.egress = this.normalizeEgressPolicy(updates.egress);
     if (updates.tags !== undefined) edge.tags = updates.tags;
+    if (updates.publicIp !== undefined) {
+      edge.publicIp = updates.publicIp === null ? undefined : this.normalizeEdgePublicIp(updates.publicIp, 4);
+    }
+    if (updates.publicIpV6 !== undefined) {
+      edge.publicIpV6 = updates.publicIpV6 === null ? undefined : this.normalizeEdgePublicIp(updates.publicIpV6, 6);
+    }
+    if (updates.mailHostname !== undefined) {
+      edge.mailHostname = updates.mailHostname === null ? undefined : this.normalizeMailHostname(updates.mailHostname);
+    }
     edge.updatedAt = Date.now();
 
     const doc = await RemoteIngressEdgeDoc.findById(id);
@@ -352,6 +405,7 @@ export class RemoteIngressManager {
       await doc.save();
     }
     this.edges.set(id, edge);
+    this.notifyEdgesChanged(`edge updated: ${edge.name}`);
     return edge;
   }
 
@@ -367,6 +421,7 @@ export class RemoteIngressManager {
       await doc.delete();
     }
     this.edges.delete(id);
+    this.notifyEdgesChanged(`edge deleted: ${id}`);
     return true;
   }
 
@@ -432,6 +487,62 @@ export class RemoteIngressManager {
     return result;
   }
 
+  /**
+   * Resolve enabled edges whose id or tags match any entry in the filter
+   * (same semantics as route-level edgeFilter in derivePortsForEdge).
+   */
+  public resolveEdgesByFilter(filter: string[]): IRemoteIngress[] {
+    if (!Array.isArray(filter) || filter.length === 0) {
+      return [];
+    }
+    const result: IRemoteIngress[] = [];
+    for (const edge of this.edges.values()) {
+      if (!edge.enabled) continue;
+      const idMatch = filter.includes(edge.id);
+      const tagMatch = edge.tags?.some((tag) => filter.includes(tag)) ?? false;
+      if (idMatch || tagMatch) {
+        result.push(edge);
+      }
+    }
+    return result;
+  }
+
+  /** Enabled edges carrying the well-known 'mail' tag. */
+  public getMailEdges(): IRemoteIngress[] {
+    return this.resolveEdgesByFilter([REMOTE_INGRESS_MAIL_TAG]);
+  }
+
+  private normalizeEdgePublicIp(value: string | undefined, family: 4 | 6): string | undefined {
+    if (value === undefined) {
+      return undefined;
+    }
+    const ip = `${value}`.trim();
+    if (!ip) {
+      return undefined;
+    }
+    if (plugins.smartnetwork.getIpVersion(ip) !== family) {
+      throw new Error(`Edge publicIp${family === 6 ? 'V6' : ''} must be a valid IPv${family} address`);
+    }
+    if (this.hubPublicIps.includes(ip)) {
+      throw new Error(`Edge public IP ${ip} equals the hub public IP — the hub IP must never front edge traffic`);
+    }
+    return ip;
+  }
+
+  private normalizeMailHostname(value: string | undefined): string | undefined {
+    if (value === undefined) {
+      return undefined;
+    }
+    const hostname = `${value}`.trim().toLowerCase().replace(/\.$/, '');
+    if (!hostname) {
+      return undefined;
+    }
+    if (hostname.length > 253 || !fqdnRegex.test(hostname)) {
+      throw new Error('Edge mailHostname must be a fully qualified domain name');
+    }
+    return hostname;
+  }
+
   private normalizeEgressPolicy(
     egress?: IRemoteIngressEgressPolicyConfig,
   ): IRemoteIngressEgressPolicyConfig | undefined {