@serve.zone/coremail 1.1.0 → 1.3.0

package/ts/classes.auth.ts

@@ -321,6 +321,83 @@ export class CoreMailAuth {
     }
   }
 
+  /**
+   * Verify an SMTP submission credential.
+   *
+   * SMTP AUTH carries no credential id, so every currently accepted verifier
+   * of the binding is tried. Verification, the argon2id code and the
+   * per-identity limiter are the socket path's: the identity key is the same
+   * `workload:<bindingId>`, so a brute force cannot escape the limit by
+   * switching surface.
+   *
+   * Returns undefined for every refusal — an unknown binding, a binding that
+   * is not active or not outbound-capable, and a wrong secret are deliberately
+   * indistinguishable to the caller.
+   */
+  public async authenticateSmtpSubmission(
+    bindingIdArg: string,
+    secretArg: string,
+  ): Promise<{
+    binding: plugins.serveZoneInterfaces.data.ICoreMailBindingDesiredState;
+    configEpoch: number;
+    credentialId: string;
+    credentialVersion: number;
+  } | undefined> {
+    let failureCode = 'INVALID_INPUT';
+    try {
+      const desiredState = await this.getActiveDesiredState();
+      const bindingId = requireCoreMailIdentifier(bindingIdArg, 'bindingId');
+      const secret = this.requireSecretShape(secretArg);
+      failureCode = 'UNKNOWN_BINDING';
+      const binding = desiredState?.bindings.find((bindingArg) =>
+        bindingArg.bindingId === bindingId
+        && bindingArg.state === 'active'
+        && bindingArg.capabilities.includes('outbound')
+      );
+      if (!desiredState || !binding) {
+        throw new CoreMailAuthenticationError();
+      }
+      failureCode = 'UNKNOWN_CREDENTIAL';
+      const accepted = binding.credentials.filter((credentialArg) =>
+        findAcceptedVerifier(
+          binding.credentials,
+          credentialArg.credentialId,
+          credentialArg.version,
+          this.now(),
+        ) !== undefined
+      );
+      if (accepted.length === 0) {
+        throw new CoreMailAuthenticationError();
+      }
+      failureCode = 'VERIFIER_REJECTED';
+      for (const verifier of accepted) {
+        try {
+          await this.verify(verifier, secret, `workload:${bindingId}`);
+        } catch (error) {
+          if (error instanceof CoreMailAuthLimitError) throw error;
+          continue;
+        }
+        return {
+          binding,
+          configEpoch: desiredState.configEpoch,
+          credentialId: verifier.credentialId,
+          credentialVersion: verifier.version,
+        };
+      }
+      throw new CoreMailAuthenticationError();
+    } catch (error) {
+      await logCoreMailFailure(
+        'auth',
+        error instanceof CoreMailAuthLimitError
+          ? 'LIMITER_EXHAUSTED'
+          : failureCode,
+        undefined,
+        { surface: 'smtp' },
+      ).catch(() => undefined);
+      return undefined;
+    }
+  }
+
   public async authenticateWorkload(
     peerArg: ICoreMailPeer,
     requestArg:

package/ts/classes.coremail.ts

@@ -14,6 +14,7 @@ import {
   type ICoreMailModels,
 } from './classes.models.js';
 import { CoreMailServer } from './classes.server.js';
+import { CoreMailSmtpSubmissionServer } from './classes.smtpsubmission.js';
 import { CoreMailStorage } from './classes.storage.js';
 import { CoreMailSubmissionService } from './classes.submissions.js';
 import { CoreMailTransferService } from './classes.transfer.js';
@@ -39,6 +40,7 @@ export class CoreMail {
   public readonly inbound: CoreMailInboundService;
   public readonly maintenance: CoreMailMaintenanceService;
   public readonly gateway: CoreMailGateway;
+  public readonly smtpSubmission: CoreMailSmtpSubmissionServer;
   public readonly server: CoreMailServer;
   public readonly outboundWorker: CoreMailOutboundWorker;
 
@@ -48,6 +50,7 @@ export class CoreMail {
   private databaseStarted = false;
   private storageStarted = false;
   private gatewayStarted = false;
+  private smtpSubmissionStarted = false;
   private serverStarted = false;
   private workerStarted = false;
   private maintenanceStarted = false;
@@ -85,21 +88,30 @@ export class CoreMail {
       this.database,
       this.models,
       this.transfers,
+      this.storage,
       configArg.replica.serviceId,
       this.desiredState.getActiveDesiredState,
       configArg.resolveRuntimeSecret,
     );
     this.gateway = new CoreMailGateway(
       configArg,
+      this.database,
       this.models,
       this.transfers,
       this.inbound,
       this.desiredState.getActiveDesiredState,
     );
+    this.smtpSubmission = new CoreMailSmtpSubmissionServer(
+      configArg,
+      this.auth,
+      this.submissions,
+      this.desiredState.getActiveDesiredState,
+    );
     this.maintenance = new CoreMailMaintenanceService(this.models, this.storage);
     this.server = server = new CoreMailServer(
       configArg,
       this.database,
+      this.models,
       this.storage,
       this.transfers,
       this.desiredState,
@@ -107,6 +119,7 @@ export class CoreMail {
       this.submissions,
       this.inbound,
       this.gateway,
+      this.smtpSubmission,
     );
     this.outboundWorker = new CoreMailOutboundWorker(
       this.models,
@@ -163,6 +176,8 @@ export class CoreMail {
     await this.desiredState.start();
     this.gateway.start();
     this.gatewayStarted = true;
+    this.smtpSubmission.start();
+    this.smtpSubmissionStarted = true;
     await this.server.start();
     this.serverStarted = true;
     this.maintenance.start();
@@ -226,6 +241,11 @@ export class CoreMail {
       () => this.maintenance.stop(),
       () => { this.maintenanceStarted = false; },
     );
+    await stopOne(
+      this.smtpSubmissionStarted,
+      () => this.smtpSubmission.stop(),
+      () => { this.smtpSubmissionStarted = false; },
+    );
     await stopOne(
       this.gatewayStarted,
       () => this.gateway.stop(),

package/ts/classes.gateway.ts

@@ -6,6 +6,13 @@ import type {
   ICoreMailConfig,
   ITransferAuthority,
 } from './interfaces.js';
+import { addPrivacySafeTypedHandler } from './coremail.errors.js';
+import { withCoreMailTransaction } from './coremail.quota.js';
+import {
+  createCoreMailStatDescriptor,
+  ensureCoreMailStatCounter,
+  incrementCoreMailStatCounter,
+} from './coremail.stats.js';
 import { logCoreMailFailure } from './coremail.log.js';
 import type { ICoreMailSubmissionRecord } from './coremail.persistence.js';
 import {
@@ -111,6 +118,7 @@ export class CoreMailGateway {
 
   public constructor(
     private readonly config: ICoreMailConfig,
+    private readonly database: plugins.smartdata.SmartdataDb,
     private readonly models: ICoreMailModels,
     private readonly transfers: CoreMailTransferService,
     private readonly inbound: CoreMailInboundService,
@@ -182,31 +190,7 @@ export class CoreMailGateway {
   ): Promise<void> {
     await this.disconnect();
     if (this.stopController.signal.aborted) return;
-    const router = new plugins.typedrequest.TypedRouter();
-    router.addTypedHandler(new plugins.typedrequest.TypedHandler<
-      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayResolveRecipients
-    >('coreMailGatewayResolveRecipients', async (requestArg) => {
-      this.requireAuthenticated(signatureArg);
-      return await this.inbound.resolveRecipients(requestArg);
-    }));
-    router.addTypedHandler(new plugins.typedrequest.TypedHandler<
-      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayPrepareInboundHandoff
-    >('coreMailGatewayPrepareInboundHandoff', async (requestArg) => {
-      this.requireAuthenticated(signatureArg);
-      return await this.inbound.prepareInboundHandoff(requestArg);
-    }));
-    router.addTypedHandler(new plugins.typedrequest.TypedHandler<
-      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayCompleteInboundHandoff
-    >('coreMailGatewayCompleteInboundHandoff', async (requestArg) => {
-      this.requireAuthenticated(signatureArg);
-      return await this.inbound.completeInboundHandoff(requestArg);
-    }));
-    router.addTypedHandler(new plugins.typedrequest.TypedHandler<
-      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayCompleteOutboundHandoff
-    >('coreMailGatewayCompleteOutboundHandoff', async (requestArg) => {
-      this.requireAuthenticated(signatureArg);
-      return await this.completeOutboundHandoff(requestArg);
-    }));
+    const router = this.createGatewayRouter(signatureArg);
     const socket = await plugins.typedsocket.TypedSocket.createClient(
       router,
       gatewayArg.endpointUrl,
@@ -235,6 +219,45 @@ export class CoreMailGateway {
     this.queueAuthentication(gatewayArg, signatureArg);
   }
 
+  /**
+   * The gateway-facing router.
+   *
+   * Every handler goes through the same privacy-safe envelope the workload
+   * router uses, so dcrouter observes the typed `ICoreMailErrorData` code —
+   * `QUOTA_EXCEEDED` versus `STATE_CONFLICT` decides whether it defers a
+   * message or rejects it — instead of an opaque error string.
+   */
+  public createGatewayRouter(
+    signatureArg: string,
+  ): plugins.typedrequest.TypedRouter {
+    const router = new plugins.typedrequest.TypedRouter();
+    addPrivacySafeTypedHandler<
+      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayResolveRecipients
+    >(router, 'gateway', 'coreMailGatewayResolveRecipients', async (requestArg) => {
+      this.requireAuthenticated(signatureArg);
+      return await this.inbound.resolveRecipients(requestArg);
+    });
+    addPrivacySafeTypedHandler<
+      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayPrepareInboundHandoff
+    >(router, 'gateway', 'coreMailGatewayPrepareInboundHandoff', async (requestArg) => {
+      this.requireAuthenticated(signatureArg);
+      return await this.inbound.prepareInboundHandoff(requestArg);
+    });
+    addPrivacySafeTypedHandler<
+      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayCompleteInboundHandoff
+    >(router, 'gateway', 'coreMailGatewayCompleteInboundHandoff', async (requestArg) => {
+      this.requireAuthenticated(signatureArg);
+      return await this.inbound.completeInboundHandoff(requestArg);
+    });
+    addPrivacySafeTypedHandler<
+      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayCompleteOutboundHandoff
+    >(router, 'gateway', 'coreMailGatewayCompleteOutboundHandoff', async (requestArg) => {
+      this.requireAuthenticated(signatureArg);
+      return await this.completeOutboundHandoff(requestArg);
+    });
+    return router;
+  }
+
   private queueAuthentication(
     gatewayArg: TGatewayDesired,
     signatureArg: string,
@@ -520,49 +543,81 @@ export class CoreMailGateway {
       ) {
         throw new Error('CoreMail gateway status chronology conflicts.');
       }
-      const result = await this.models.Submission.exact.transition({
-        current: submission,
-        change: (modelArg) => {
-          modelArg.transportMessageId = statusArg.transportMessageId;
-          modelArg.acceptedAt = acceptedAt;
-          modelArg.state = statusArg.state;
-          modelArg.attempts = Math.max(
-            submission.attempts,
-            statusArg.attempts,
-          );
-          modelArg.gatewayStatusUpdatedAt = statusArg.updatedAt;
-          if (nextAttemptAt === undefined) {
-            delete modelArg.nextAttemptAt;
-          } else {
-            modelArg.nextAttemptAt = nextAttemptAt;
-          }
-          if (workerRetryAt === undefined) {
-            delete modelArg.workerRetryAt;
-          } else {
-            modelArg.workerRetryAt = workerRetryAt;
-          }
-          if (statusArg.state === 'delivered') {
-            modelArg.deliveredAt = statusArg.deliveredAt;
-          } else {
-            delete modelArg.deliveredAt;
-          }
-          if (statusArg.terminalAt === undefined) {
-            delete modelArg.terminalAt;
-          } else {
-            modelArg.terminalAt = statusArg.terminalAt;
-          }
-          if (statusArg.error === undefined) {
-            delete modelArg.error;
-          } else {
-            modelArg.error = statusArg.error;
+      // A terminal state is reached at most once (the loop returns early once
+      // the record is terminal) and a deferral counts only when the gateway
+      // status is strictly newer than the one already recorded, so replaying
+      // one status never counts twice.
+      const statKind = terminal
+        ? statusArg.state as 'delivered' | 'failed' | 'deadLettered'
+        : statusArg.state === 'deferred'
+            && (
+              submission.gatewayStatusUpdatedAt === undefined
+              || statusArg.updatedAt > submission.gatewayStatusUpdatedAt
+            )
+          ? 'deferred' as const
+          : undefined;
+      const statDescriptor = createCoreMailStatDescriptor(submission, updatedAt);
+      if (statKind) {
+        await ensureCoreMailStatCounter(this.models, statDescriptor);
+      }
+      const result = await withCoreMailTransaction(
+        this.database,
+        async (transactionArg) => {
+          const transitioned = await this.models.Submission.exact.transition({
+            current: submission,
+            change: (modelArg) => {
+              modelArg.transportMessageId = statusArg.transportMessageId;
+              modelArg.acceptedAt = acceptedAt;
+              modelArg.state = statusArg.state;
+              modelArg.attempts = Math.max(
+                submission.attempts,
+                statusArg.attempts,
+              );
+              modelArg.gatewayStatusUpdatedAt = statusArg.updatedAt;
+              if (nextAttemptAt === undefined) {
+                delete modelArg.nextAttemptAt;
+              } else {
+                modelArg.nextAttemptAt = nextAttemptAt;
+              }
+              if (workerRetryAt === undefined) {
+                delete modelArg.workerRetryAt;
+              } else {
+                modelArg.workerRetryAt = workerRetryAt;
+              }
+              if (statusArg.state === 'delivered') {
+                modelArg.deliveredAt = statusArg.deliveredAt;
+              } else {
+                delete modelArg.deliveredAt;
+              }
+              if (statusArg.terminalAt === undefined) {
+                delete modelArg.terminalAt;
+              } else {
+                modelArg.terminalAt = statusArg.terminalAt;
+              }
+              if (statusArg.error === undefined) {
+                delete modelArg.error;
+              } else {
+                modelArg.error = statusArg.error;
+              }
+              modelArg.updatedAt = updatedAt;
+              modelArg.revision += 1;
+              delete modelArg.leaseOwnerTaskId;
+              delete modelArg.leaseToken;
+              delete modelArg.leaseExpiresAt;
+            },
+              }, { session: transactionArg });
+          if (transitioned.status === 'transitioned' && statKind) {
+            await incrementCoreMailStatCounter(
+              this.models,
+              statDescriptor,
+              statKind,
+              updatedAt,
+              transactionArg,
+            );
           }
-          modelArg.updatedAt = updatedAt;
-          modelArg.revision += 1;
-          delete modelArg.leaseOwnerTaskId;
-          delete modelArg.leaseToken;
-          delete modelArg.leaseExpiresAt;
+          return transitioned;
         },
-      });
+      );
       if (result.status === 'transitioned') {
         return result.document;
       }

package/ts/classes.inbound.ts

@@ -3,15 +3,24 @@ import { type ICoreMailModels } from './classes.models.js';
 import {
   createOpaqueToken,
   createSha256,
+  type CoreMailStorage,
 } from './classes.storage.js';
 import { CoreMailTransferService } from './classes.transfer.js';
 import {
-  normalizeCoreMailConnectionInfo,
   normalizeCoreMailEnvelope,
-  normalizeCoreMailGatewayMessage,
   normalizeCoreMailMailboxAddress,
 } from './coremail.validation.js';
 import { CoreMailInboundCursorCodec } from './coremail.cursor.js';
+import { logCoreMailFailure } from './coremail.log.js';
+import {
+  coreMailHeaderScanBytes,
+  parseCoreMailMessageHeaders,
+} from './coremail.mime.js';
+import {
+  createCoreMailStatDescriptor,
+  ensureCoreMailStatCounter,
+  incrementCoreMailStatCounter,
+} from './coremail.stats.js';
 import {
   createCoreMailPendingQuotaDescriptor,
   decrementCoreMailQuotaCounter,
@@ -100,7 +109,13 @@ const toPublicDelivery = (
     : {}),
 });
 
-const createEnvelopeDigest = (
+/**
+ * Routing-handle binding digest over the exact normalized envelope and source.
+ * Exported so a regression test can pin the byte layout: the normalizers were
+ * moved to the upstream contract package, and a silently different shape here
+ * would invalidate every issued handle.
+ */
+export const createEnvelopeDigest = (
   envelopeArg: plugins.serveZoneInterfaces.data.ICoreMailEnvelope,
   sourceArg: plugins.serveZoneInterfaces.data.IMailConnectionInfo,
 ) => createSha256(JSON.stringify({
@@ -137,6 +152,7 @@ export class CoreMailInboundService {
     private readonly database: plugins.smartdata.SmartdataDb,
     private readonly models: ICoreMailModels,
     private readonly transfers: CoreMailTransferService,
+    private readonly storage: CoreMailStorage,
     private readonly serviceId: string,
     private readonly getDesiredState: () => Promise<TDesiredState | null>,
     resolveRuntimeSecretArg: (secretKeyArg: string) => string,
@@ -163,7 +179,7 @@ export class CoreMailInboundService {
       throw new Error('CoreMail gateway recipients are invalid.');
     }
     const envelope = normalizeCoreMailEnvelope(requestArg.envelope);
-    const source = normalizeCoreMailConnectionInfo(requestArg.source);
+    const source = plugins.serveZoneInterfaces.data.normalizeCoreMailConnectionInfo(requestArg.source);
     const envelopeDigest = createEnvelopeDigest(envelope, source);
     const recipients = requestArg.recipients.map(normalizeCoreMailMailboxAddress);
     if (new Set(recipients).size !== recipients.length) {
@@ -270,8 +286,8 @@ export class CoreMailInboundService {
     if (new Set(routingHandles).size !== routingHandles.length) {
       throw new Error('CoreMail inbound handoff contains duplicate routing handles.');
     }
-    const message = normalizeCoreMailGatewayMessage(requestArg.message);
-    const source = normalizeCoreMailConnectionInfo(requestArg.source);
+    const message = plugins.serveZoneInterfaces.data.normalizeCoreMailGatewayMessage(requestArg.message);
+    const source = plugins.serveZoneInterfaces.data.normalizeCoreMailConnectionInfo(requestArg.source);
     const envelopeDigest = createEnvelopeDigest(message.envelope, source);
     const existing = await this.models.InboundHandoff.exact.findStoredOne({
       transportDeliveryId,
@@ -472,6 +488,25 @@ export class CoreMailInboundService {
       grouped.set(key, group);
     }
     const groups = [...grouped.values()];
+    // The gateway message descriptor cannot carry Message-ID or Subject, so
+    // both are read from the stored bytes. A scan failure must never lose a
+    // delivery: the fields stay absent.
+    let headers: { messageId?: string; subject?: string } = {};
+    try {
+      headers = parseCoreMailMessageHeaders(
+        await this.storage.readObjectHeadBytes(
+          handoff.objectKey,
+          handoff.message.rawMime.lengthBytes,
+          handoff.message.rawMime.sha256,
+          plugins.serveZoneInterfaces.data.coreMailLimits.serializedMimeBytes,
+          coreMailHeaderScanBytes,
+        ),
+      );
+    } catch (error) {
+      await logCoreMailFailure('gateway', 'HEADER_SCAN_FAILED', error, {
+        handoffId: handoff.handoffId,
+      }).catch(() => undefined);
+    }
     const candidates: ICoreMailInboundDeliveryRecord[] = [];
     for (const group of groups) {
       const authority = group[0];
@@ -483,6 +518,10 @@ export class CoreMailInboundService {
       const quota = createCoreMailPendingQuotaDescriptor(authority);
       await ensureCoreMailQuotaCounter(this.models, quota);
       const receivedAt = this.now();
+      await ensureCoreMailStatCounter(
+        this.models,
+        createCoreMailStatDescriptor(authority, receivedAt),
+      );
       candidates.push({
         deliveryId: plugins.nodeCrypto.randomUUID(),
         tenantId: authority.tenantId,
@@ -499,6 +538,10 @@ export class CoreMailInboundService {
         rawMimeSha256: handoff.message.rawMime.sha256,
         rawMimeLengthBytes: handoff.message.rawMime.lengthBytes,
         rawMimeObjectKey: handoff.objectKey,
+        ...(headers.messageId === undefined
+          ? {}
+          : { messageId: headers.messageId }),
+        ...(headers.subject === undefined ? {} : { subject: headers.subject }),
         receivedAt,
         updatedAt: receivedAt,
       });
@@ -541,6 +584,15 @@ export class CoreMailInboundService {
               if (inserted.status === 'conflict') {
                 throw new Error('CoreMail inbound delivery identity conflict.');
               }
+              // Counted with the insert, so a replayed completion that finds
+              // the delivery already present counts nothing.
+              await incrementCoreMailStatCounter(
+                this.models,
+                createCoreMailStatDescriptor(authority, candidate.receivedAt),
+                'received',
+                candidate.receivedAt,
+                transactionArg,
+              );
             }
             for (const originalHandle of group) {
               const handle = await this.models.RecipientHandle.exact.findStoredOne(
@@ -833,6 +885,8 @@ export class CoreMailInboundService {
       bindingRevision: sessionArg.bindingRevision,
     });
     await ensureCoreMailQuotaCounter(this.models, quota);
+    const statDescriptor = createCoreMailStatDescriptor(sessionArg, updatedAt);
+    await ensureCoreMailStatCounter(this.models, statDescriptor);
     const transactionResult = await withCoreMailTransaction(
       this.database,
       async (transactionArg) => {
@@ -872,6 +926,17 @@ export class CoreMailInboundService {
         if (update.status !== 'transitioned') {
           throw new Error('CoreMail inbound acknowledgement fence changed.');
         }
+        // Counted with the acknowledgement transition; the replay branch above
+        // returns before reaching this point.
+        await incrementCoreMailStatCounter(
+          this.models,
+          statDescriptor,
+          requestArg.outcome === 'processed'
+            ? 'acknowledgedProcessed'
+            : 'acknowledgedDiscarded',
+          updatedAt,
+          transactionArg,
+        );
         return { replayed: false };
       },
     );

package/ts/classes.maintenance.ts

@@ -2,6 +2,7 @@ import * as plugins from './plugins.js';
 import type { ICoreMailModels } from './classes.models.js';
 import type { CoreMailStorage } from './classes.storage.js';
 import { logCoreMailFailure } from './coremail.log.js';
+import { coreMailStatRetentionMs } from './coremail.stats.js';
 
 const sweepIntervalMs = 60_000;
 const batchSize = 100;
@@ -274,6 +275,19 @@ export class CoreMailMaintenanceService {
       }
       if (handles.length < batchSize) break;
     }
+    const statCutoff = now - coreMailStatRetentionMs;
+    while (!signalArg?.aborted) {
+      const statCounters = await this.models.StatCounter.exact.findStored({
+        filter: { dayUtc: { $lte: statCutoff } },
+        sort: { dayUtc: 1, statId: 1 },
+        limit: batchSize,
+      });
+      for (const statCounter of statCounters) {
+        signalArg?.throwIfAborted();
+        await this.models.StatCounter.exact.delete({ current: statCounter });
+      }
+      if (statCounters.length < batchSize) break;
+    }
     while (!signalArg?.aborted) {
       const counters = await this.models.QuotaCounter.exact.findStored({
         filter: {