@serve.zone/dcrouter 17.3.0 → 17.4.1

package/ts/classes.dcrouter.ts

@@ -33,6 +33,7 @@ import { RouteConfigManager, ApiTokenManager, GatewayClientManager, ReferenceRes
 import type { TVpnClientAllowEntry } from './config/classes.route-config-manager.js';
 import { SecurityLogger, ContentScanner, IPReputationChecker, SecurityPolicyManager, RoutePolicyAugmenter } from './security/index.js';
 import { type IHttp3Config, augmentRoutesWithHttp3 } from './http3/index.js';
+import { applyDefaultInboundPolicy } from './email/inbound-policy.js';
 import { DnsManager } from './dns/manager.dns.js';
 import { DnsServerRuntime } from './dns/classes.dns-server-runtime.js';
 import { GatewayRouteDnsReconciler } from './dns/classes.gateway-route-dns-reconciler.js';
@@ -1770,6 +1771,7 @@ export class DcRouter {
     let emailConfig: IUnifiedEmailServerOptions = await this.workAppMailManager.applyStoredIdentitiesToEmailConfig({
       ...this.options.emailConfig,
       ports: mappedEmailPorts,
+      domains: applyDefaultInboundPolicy(this.options.emailConfig.domains),
       dkimKeyProvisioning: 'caller-managed',
       persistRoutes: this.options.emailConfig.persistRoutes ?? false,
       queue: queueOptions,
@@ -1812,6 +1814,13 @@ export class DcRouter {
             smtpMessage: configuredDecision?.smtpMessage ?? dcrouterDecision.smtpMessage,
           };
         },
+        onAcceptEnvelope: async (context) => {
+          const emailServer = this.emailServer;
+          if (!emailServer) {
+            throw new Error('Email server is not available for durable envelope acceptance');
+          }
+          await this.acceptedEmailSpool.acceptEnvelope(context, emailServer);
+        },
       },
     });
     
@@ -1827,7 +1836,7 @@ export class DcRouter {
       )) {
         emailConfig = await this.workAppMailManager.applyStoredIdentitiesToEmailConfig({
           ...emailConfig,
-          domains: [...(this.options.emailConfig?.domains || [])],
+          domains: applyDefaultInboundPolicy(this.options.emailConfig?.domains),
         });
         // DkimManager captures its DomainRegistry at construction. Rebuild the
         // unstarted server so per-domain repair failures are absent from

package/ts/db/documents/classes.cached.email.ts

@@ -6,7 +6,7 @@ const TTL = plugins.smartdata.smartdataTtlValues;
 /**
  * Email status in the cache
  */
-export type TCachedEmailStatus = 'pending' | 'processing' | 'queued' | 'delivered' | 'failed' | 'deferred';
+export type TCachedEmailStatus = 'pending' | 'processing' | 'queued' | 'delivered' | 'failed' | 'deferred' | 'stored';
 
 /**
  * Direction of an accepted email: 'inbound' = received from a remote peer,
@@ -141,6 +141,14 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
   @plugins.smartdata.svDb()
   public status!: TCachedEmailStatus;
 
+  /** Attachment count derived from the parsed message at acceptance. */
+  @plugins.smartdata.svDb()
+  public attachmentCount?: number;
+
+  /** JSON-serialized inbound SPF/DKIM/DMARC verdicts from the SMTP session. */
+  @plugins.smartdata.svDb()
+  public inboundSecurityResults?: string;
+
   /**
    * Number of delivery attempts
    */
@@ -344,6 +352,18 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
     this.deliveredAt = new Date();
   }
 
+  /**
+   * Mark as terminally stored (catch-all inbound): visible in the email log,
+   * pruned after the retention window.
+   */
+  public markStored(retentionMs: number): void {
+    this.status = 'stored';
+    this.deliveredAt = new Date();
+    // Base-class TTL: the CacheCleaner deletes expired rows and their raw
+    // payloads via beforeDeleteCachedEmail.
+    this.setTTL(retentionMs);
+  }
+
   /**
    * Mark as failed with error
    */

package/ts/email/classes.accepted-email-spool.ts

@@ -7,6 +7,7 @@ import type {
 } from '../db/documents/classes.cached.email.js';
 import type {
   Email,
+  IAcceptEnvelopeContext,
   IExtendedSmtpSession,
   IMessageAcceptanceContext,
   IMessageAcceptanceDecision,
@@ -20,6 +21,8 @@ const ACCEPTED_EMAIL_RETRY_DELAY_MS = 5 * 60_000;
 const ACCEPTED_EMAIL_QUEUE_LEASE_MS = 30 * 60_000;
 const ACCEPTED_EMAIL_SPOOL_BATCH_SIZE = 25;
 const ACCEPTED_EMAIL_STOP_DRAIN_TIMEOUT_MS = 30_000;
+/** Retention for catch-all stored inbound mail (30 days). */
+const INBOUND_STORE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
 
 export type TSmartMtaQueueItemLike = {
   id?: string;
@@ -252,6 +255,114 @@ export class AcceptedEmailSpool {
     };
   }
 
+  /**
+   * Durable acceptance for envelopes containing at least one catch-all store
+   * route: anchors the exact raw message + metadata as a CachedEmail before
+   * SMTP 250, then dispatches any non-store recipients through SmartMTA's
+   * exact-plan executor. Store recipients are fulfilled by the anchor itself.
+   */
+  public async acceptEnvelope(
+    context: IAcceptEnvelopeContext,
+    emailServer: UnifiedEmailServer,
+  ): Promise<void> {
+    if (!this.dcRouterRef.dcRouterDb?.isReady()) {
+      throw new Error('DcRouterDb is not available for durable envelope acceptance');
+    }
+    this.throwIfMessageAcceptanceAborted(context.abortSignal);
+
+    const email = context.email;
+    const session = context.session;
+    const cachedEmail = CachedEmail.createNew();
+    this.removeHeader(email.headers, DCROUTER_CACHE_ID_HEADER);
+    email.headers[DCROUTER_CACHE_ID_HEADER] = cachedEmail.id;
+    cachedEmail.messageId = email.headers['Message-ID'] || email.headers['message-id'] || cachedEmail.id;
+    cachedEmail.from = context.envelope.mailFrom || email.from || '';
+    cachedEmail.to = context.envelope.rcptTo.length > 0
+      ? [...context.envelope.rcptTo]
+      : Array.isArray(email.to) ? email.to : [];
+    cachedEmail.cc = Array.isArray(email.cc) ? email.cc : [];
+    cachedEmail.bcc = Array.isArray(email.bcc) ? email.bcc : [];
+    cachedEmail.subject = email.subject || '';
+    cachedEmail.attachmentCount = context.attachmentCount;
+    if (context.securityResults) {
+      cachedEmail.inboundSecurityResults = JSON.stringify(context.securityResults);
+    }
+
+    // Build the exact per-recipient dispatch plan before durably committing,
+    // so an unplannable envelope is refused instead of half-anchored.
+    const idempotencyKeys = context.resolvedRecipientRoutes.map(
+      (resolution) => `${cachedEmail.id}:${resolution.recipient}`,
+    );
+    const plan = emailServer.createAcceptedEnvelopeDispatchPlan(context, idempotencyKeys);
+    const nonStoreEntries = plan.filter((entry) => entry.action.type !== 'store');
+
+    const persistedRawMessage = this.setDcRouterCacheIdHeader(
+      context.rawMessage.toString('utf8'),
+      cachedEmail.id,
+    );
+    await this.persistRawMessage(cachedEmail, persistedRawMessage);
+    cachedEmail.direction = session.authenticated ? 'outbound' : 'inbound';
+    cachedEmail.acceptedAt = Date.now();
+    cachedEmail.markStored(INBOUND_STORE_RETENTION_MS);
+    cachedEmail.routeData = JSON.stringify({
+      acceptedAt: new Date().toISOString(),
+      acceptance: 'durable-envelope',
+      recipientPlans: plan.map((entry) => ({
+        recipient: entry.recipient,
+        routeName: entry.routeName,
+        source: entry.source,
+        actionType: entry.action.type,
+      })),
+      session: {
+        id: session.id,
+        remoteAddress: session.remoteAddress,
+        clientHostname: session.clientHostname,
+        secure: !!session.secure,
+        authenticated: !!session.authenticated,
+        user: session.user,
+        envelope: context.envelope,
+      },
+    });
+    cachedEmail.updateSenderDomain();
+    cachedEmail.updateRecipientDomains();
+    try {
+      await cachedEmail.save();
+      await this.notifyEmailQueuePersisted(cachedEmail, 'envelope-stored');
+    } catch (error) {
+      await this.deleteRawMessage(cachedEmail).catch(() => undefined);
+      throw error;
+    }
+    if (context.abortSignal?.aborted) {
+      cachedEmail.markFailed('Envelope acceptance aborted before SMTP success');
+      await cachedEmail.save();
+      await this.notifyEmailQueuePersisted(cachedEmail, 'envelope-acceptance-aborted');
+      throw new Error('Envelope acceptance aborted before SMTP success');
+    }
+
+    if (nonStoreEntries.length > 0) {
+      const dispatchResult = await emailServer.dispatchAcceptedEnvelope(
+        context.rawMessage,
+        plan,
+        {
+          mailFrom: context.envelope.mailFrom,
+          session: {
+            id: session.id,
+            remoteAddress: session.remoteAddress,
+            clientHostname: session.clientHostname,
+            secure: !!session.secure,
+            authenticated: !!session.authenticated,
+          },
+        },
+      );
+      const failed = dispatchResult.results.filter(
+        (result) => result.status === 'failed' || result.status === 'rejected',
+      );
+      if (failed.length > 0) {
+        logger.log('warn', `Durable envelope ${cachedEmail.id}: ${failed.length}/${dispatchResult.results.length} non-store recipients failed dispatch: ${failed.map((result) => `${result.recipient}=${result.message || result.status}`).join('; ')}`);
+      }
+    }
+  }
+
   /** Start the interval-driven spool processor and trigger an immediate run. */
   public start(): void {
     this.clearSpoolTimer();

package/ts/email/classes.email-domain.manager.ts

@@ -16,6 +16,7 @@ import {
   MAIL_DNS_MAX_RETRY_INTERVAL_MS,
   MAIL_DNS_RETRY_INTERVAL_MS,
 } from './classes.mail-dns-sync.js';
+import { applyDefaultInboundPolicy } from './inbound-policy.js';
 import { projectMailDnsIntentStatus } from './mail-dns-status.js';
 
 export interface IEmailDomainManagerActionResult {
@@ -884,7 +885,7 @@ export class EmailDomainManager {
       }
       mergedDomains.set(key, managedConfig);
     }
-    const domains = [...mergedDomains.values()];
+    const domains = applyDefaultInboundPolicy([...mergedDomains.values()]);
     this.dcRouter.options.emailConfig.domains = domains;
     this.dcRouter.emailServer?.updateOptions({ domains });
   }

package/ts/email/classes.smartmta-storage-manager.ts

@@ -1,5 +1,9 @@
 import * as plugins from '../plugins.js';
-import type { IStorageManager } from '@push.rocks/smartmta';
+import type {
+  IStorageManager,
+  IStorageListPageOptions,
+  IStorageListPageResult,
+} from '@push.rocks/smartmta';
 import {
   getSmartMtaTextNamespace,
   hashSmartMtaStorageKey,
@@ -98,6 +102,58 @@ export class SmartMtaStorageManager implements IStorageManager {
       .sort();
   }
 
+  /**
+   * Bounded, deletion-stable page listing: seeks strictly beyond the last
+   * returned key, so tokens stay valid while pages are deleted and an
+   * otherwise stable prefix is always exhausted.
+   */
+  public async listPage(
+    prefix: string,
+    options: IStorageListPageOptions,
+  ): Promise<IStorageListPageResult> {
+    const normalizedPrefix = normalizeSmartMtaStorageKey(prefix);
+    getSmartMtaTextNamespace(normalizedPrefix);
+    const requestedLimit = Number.isSafeInteger(options?.limit) && options.limit > 0
+      ? options.limit
+      : 100;
+    const limit = Math.min(requestedLimit, 1000);
+    let seekAfter: string | undefined;
+    if (options?.continuationToken) {
+      const decoded = Buffer.from(options.continuationToken, 'base64url').toString('utf8');
+      if (!decoded.startsWith(normalizedPrefix)) {
+        throw new Error('SmartMTA storage continuation token does not match the requested prefix');
+      }
+      seekAfter = decoded;
+    }
+
+    const documents = await this.collection
+      .find({
+        key: {
+          ...(seekAfter ? { $gt: seekAfter } : { $gte: normalizedPrefix }),
+          $lt: `${normalizedPrefix}￿`,
+        },
+      })
+      .sort({ key: 1 })
+      .limit(limit + 1)
+      .toArray();
+
+    const hasMore = documents.length > limit;
+    const pageDocuments = documents.slice(0, limit);
+    const keys = pageDocuments
+      .map((document: ISmartMtaStorageDocument) => this.validateDocument(document).key)
+      .filter((key: string) => key.startsWith(normalizedPrefix));
+    const lastKey = pageDocuments.length > 0
+      ? (pageDocuments[pageDocuments.length - 1] as ISmartMtaStorageDocument).key
+      : undefined;
+
+    return {
+      keys,
+      ...(hasMore && lastKey
+        ? { continuationToken: Buffer.from(lastKey, 'utf8').toString('base64url') }
+        : {}),
+    };
+  }
+
   public async delete(key: string): Promise<void> {
     const normalizedKey = normalizeSmartMtaStorageKey(key);
     getSmartMtaTextNamespace(normalizedKey);

package/ts/email/inbound-policy.ts

@@ -0,0 +1,25 @@
+/**
+ * Org default inbound policy: every email domain accepts inbound mail via
+ * catch-all store with bounded retention, so replies to platform-sent mail
+ * are never bounced by omission. Explicit SmartMTA routes always outrank
+ * this no-route fallback; explicit per-domain policies are preserved.
+ */
+
+export interface IInboundPolicyLike {
+  defaultAction: 'store' | 'reject';
+  retentionDays: number;
+}
+
+export const DEFAULT_INBOUND_POLICY: IInboundPolicyLike = Object.freeze({
+  defaultAction: 'store',
+  retentionDays: 30,
+});
+
+export function applyDefaultInboundPolicy<T extends { inboundPolicy?: IInboundPolicyLike }>(
+  domainsArg: T[] | undefined,
+): T[] {
+  return (domainsArg || []).map((domain) => ({
+    ...domain,
+    inboundPolicy: domain.inboundPolicy ?? { ...DEFAULT_INBOUND_POLICY },
+  }));
+}

package/ts/email/smartmta-storage-policy.ts

@@ -11,6 +11,8 @@ export const smartMtaTextNamespaces: readonly ISmartMtaTextNamespace[] = Object.
   { prefix: '/email/bounces/', maxBytes: 8 * 1024 * 1024 },
   { prefix: '/email/templates/', maxBytes: 4 * 1024 * 1024 },
   { prefix: '/email/routes/config.json', maxBytes: 4 * 1024 * 1024 },
+  // SmartMTA 8.1+ accepted-envelope dispatch bookkeeping (idempotent replay records).
+  { prefix: '/email/accepted-envelope-dispatch/', maxBytes: 64 * 1024 },
   { prefix: '/security/ip-reputation-cache.json', maxBytes: 8 * 1024 * 1024 },
   { prefix: '/workhosters/mail-identities.json', maxBytes: 4 * 1024 * 1024 },
 ]);

package/ts/opsserver/handlers/email-ops.handler.ts

@@ -197,14 +197,7 @@ export class EmailOpsHandler {
         authMethod: '',
         authUser: this.extractSessionUser(session),
       },
-      authenticationResults: {
-        spf: 'none',
-        spfDomain: '',
-        dkim: 'none',
-        dkimDomain: '',
-        dmarc: 'none',
-        dmarcPolicy: '',
-      },
+      authenticationResults: this.mapInboundSecurityResults(doc.inboundSecurityResults),
       rejectionReason: doc.status === 'failed' ? doc.lastError : undefined,
       bounceMessage: doc.status === 'failed' ? doc.lastError : undefined,
       headers,
@@ -388,6 +381,46 @@ export class EmailOpsHandler {
   /**
    * Map queue status to catalog TEmailStatus
    */
+  /**
+   * Maps SmartMTA inbound SPF/DKIM/DMARC verdicts (persisted as JSON on the
+   * cached email) into the catalog authenticationResults shape. Falls back to
+   * all-'none' when no verdicts were captured.
+   */
+  private mapInboundSecurityResults(serializedArg?: string): interfaces.requests.IEmailDetail['authenticationResults'] {
+    const noneResult: interfaces.requests.IEmailDetail['authenticationResults'] = {
+      spf: 'none',
+      spfDomain: '',
+      dkim: 'none',
+      dkimDomain: '',
+      dmarc: 'none',
+      dmarcPolicy: '',
+    };
+    if (!serializedArg) return noneResult;
+    try {
+      const parsed = JSON.parse(serializedArg) as {
+        spf?: { result?: string; domain?: string } | null;
+        dkim?: Array<{ is_valid?: boolean; domain?: string | null; status?: string }> | null;
+        dmarc?: { passed?: boolean; policy?: string; domain?: string } | null;
+      };
+      const spfResult = (parsed.spf?.result || '').toLowerCase();
+      const spf = (['pass', 'fail', 'softfail', 'neutral'] as const).find((value) => value === spfResult) || 'none';
+      const dkimSignatures = Array.isArray(parsed.dkim) ? parsed.dkim : [];
+      const validSignature = dkimSignatures.find((signature) => signature?.is_valid);
+      const dkim = validSignature ? 'pass' : dkimSignatures.length > 0 ? 'fail' : 'none';
+      const dmarc = parsed.dmarc ? (parsed.dmarc.passed ? 'pass' : 'fail') : 'none';
+      return {
+        spf,
+        spfDomain: parsed.spf?.domain || '',
+        dkim,
+        dkimDomain: validSignature?.domain || dkimSignatures[0]?.domain || '',
+        dmarc,
+        dmarcPolicy: parsed.dmarc?.policy || '',
+      };
+    } catch {
+      return noneResult;
+    }
+  }
+
   private mapStatus(queueStatus: string): interfaces.requests.TEmailStatus {
     switch (queueStatus) {
       case 'pending':
@@ -395,6 +428,7 @@ export class EmailOpsHandler {
       case 'queued':
         return 'pending';
       case 'delivered':
+      case 'stored':
         return 'delivered';
       case 'failed':
         return 'bounced';

package/ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '17.3.0',
+  version: '17.4.1',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }