@serve.zone/coremail 1.2.0 → 32.0.0

package/ts/classes.inbound.ts

@@ -3,15 +3,24 @@ import { type ICoreMailModels } from './classes.models.js';
 import {
   createOpaqueToken,
   createSha256,
+  type CoreMailStorage,
 } from './classes.storage.js';
 import { CoreMailTransferService } from './classes.transfer.js';
 import {
-  normalizeCoreMailConnectionInfo,
   normalizeCoreMailEnvelope,
-  normalizeCoreMailGatewayMessage,
   normalizeCoreMailMailboxAddress,
 } from './coremail.validation.js';
 import { CoreMailInboundCursorCodec } from './coremail.cursor.js';
+import { logCoreMailFailure } from './coremail.log.js';
+import {
+  coreMailHeaderScanBytes,
+  parseCoreMailMessageHeaders,
+} from './coremail.mime.js';
+import {
+  createCoreMailStatDescriptor,
+  ensureCoreMailStatCounter,
+  incrementCoreMailStatCounter,
+} from './coremail.stats.js';
 import {
   createCoreMailPendingQuotaDescriptor,
   decrementCoreMailQuotaCounter,
@@ -100,7 +109,13 @@ const toPublicDelivery = (
     : {}),
 });
 
-const createEnvelopeDigest = (
+/**
+ * Routing-handle binding digest over the exact normalized envelope and source.
+ * Exported so a regression test can pin the byte layout: the normalizers were
+ * moved to the upstream contract package, and a silently different shape here
+ * would invalidate every issued handle.
+ */
+export const createEnvelopeDigest = (
   envelopeArg: plugins.serveZoneInterfaces.data.ICoreMailEnvelope,
   sourceArg: plugins.serveZoneInterfaces.data.IMailConnectionInfo,
 ) => createSha256(JSON.stringify({
@@ -137,6 +152,7 @@ export class CoreMailInboundService {
     private readonly database: plugins.smartdata.SmartdataDb,
     private readonly models: ICoreMailModels,
     private readonly transfers: CoreMailTransferService,
+    private readonly storage: CoreMailStorage,
     private readonly serviceId: string,
     private readonly getDesiredState: () => Promise<TDesiredState | null>,
     resolveRuntimeSecretArg: (secretKeyArg: string) => string,
@@ -163,7 +179,7 @@ export class CoreMailInboundService {
       throw new Error('CoreMail gateway recipients are invalid.');
     }
     const envelope = normalizeCoreMailEnvelope(requestArg.envelope);
-    const source = normalizeCoreMailConnectionInfo(requestArg.source);
+    const source = plugins.serveZoneInterfaces.data.normalizeCoreMailConnectionInfo(requestArg.source);
     const envelopeDigest = createEnvelopeDigest(envelope, source);
     const recipients = requestArg.recipients.map(normalizeCoreMailMailboxAddress);
     if (new Set(recipients).size !== recipients.length) {
@@ -270,8 +286,8 @@ export class CoreMailInboundService {
     if (new Set(routingHandles).size !== routingHandles.length) {
       throw new Error('CoreMail inbound handoff contains duplicate routing handles.');
     }
-    const message = normalizeCoreMailGatewayMessage(requestArg.message);
-    const source = normalizeCoreMailConnectionInfo(requestArg.source);
+    const message = plugins.serveZoneInterfaces.data.normalizeCoreMailGatewayMessage(requestArg.message);
+    const source = plugins.serveZoneInterfaces.data.normalizeCoreMailConnectionInfo(requestArg.source);
     const envelopeDigest = createEnvelopeDigest(message.envelope, source);
     const existing = await this.models.InboundHandoff.exact.findStoredOne({
       transportDeliveryId,
@@ -472,6 +488,25 @@ export class CoreMailInboundService {
       grouped.set(key, group);
     }
     const groups = [...grouped.values()];
+    // The gateway message descriptor cannot carry Message-ID or Subject, so
+    // both are read from the stored bytes. A scan failure must never lose a
+    // delivery: the fields stay absent.
+    let headers: { messageId?: string; subject?: string } = {};
+    try {
+      headers = parseCoreMailMessageHeaders(
+        await this.storage.readObjectHeadBytes(
+          handoff.objectKey,
+          handoff.message.rawMime.lengthBytes,
+          handoff.message.rawMime.sha256,
+          plugins.serveZoneInterfaces.data.coreMailLimits.serializedMimeBytes,
+          coreMailHeaderScanBytes,
+        ),
+      );
+    } catch (error) {
+      await logCoreMailFailure('gateway', 'HEADER_SCAN_FAILED', error, {
+        handoffId: handoff.handoffId,
+      }).catch(() => undefined);
+    }
     const candidates: ICoreMailInboundDeliveryRecord[] = [];
     for (const group of groups) {
       const authority = group[0];
@@ -483,6 +518,10 @@ export class CoreMailInboundService {
       const quota = createCoreMailPendingQuotaDescriptor(authority);
       await ensureCoreMailQuotaCounter(this.models, quota);
       const receivedAt = this.now();
+      await ensureCoreMailStatCounter(
+        this.models,
+        createCoreMailStatDescriptor(authority, receivedAt),
+      );
       candidates.push({
         deliveryId: plugins.nodeCrypto.randomUUID(),
         tenantId: authority.tenantId,
@@ -499,6 +538,10 @@ export class CoreMailInboundService {
         rawMimeSha256: handoff.message.rawMime.sha256,
         rawMimeLengthBytes: handoff.message.rawMime.lengthBytes,
         rawMimeObjectKey: handoff.objectKey,
+        ...(headers.messageId === undefined
+          ? {}
+          : { messageId: headers.messageId }),
+        ...(headers.subject === undefined ? {} : { subject: headers.subject }),
         receivedAt,
         updatedAt: receivedAt,
       });
@@ -541,6 +584,15 @@ export class CoreMailInboundService {
               if (inserted.status === 'conflict') {
                 throw new Error('CoreMail inbound delivery identity conflict.');
               }
+              // Counted with the insert, so a replayed completion that finds
+              // the delivery already present counts nothing.
+              await incrementCoreMailStatCounter(
+                this.models,
+                createCoreMailStatDescriptor(authority, candidate.receivedAt),
+                'received',
+                candidate.receivedAt,
+                transactionArg,
+              );
             }
             for (const originalHandle of group) {
               const handle = await this.models.RecipientHandle.exact.findStoredOne(
@@ -833,6 +885,8 @@ export class CoreMailInboundService {
       bindingRevision: sessionArg.bindingRevision,
     });
     await ensureCoreMailQuotaCounter(this.models, quota);
+    const statDescriptor = createCoreMailStatDescriptor(sessionArg, updatedAt);
+    await ensureCoreMailStatCounter(this.models, statDescriptor);
     const transactionResult = await withCoreMailTransaction(
       this.database,
       async (transactionArg) => {
@@ -872,6 +926,17 @@ export class CoreMailInboundService {
         if (update.status !== 'transitioned') {
           throw new Error('CoreMail inbound acknowledgement fence changed.');
         }
+        // Counted with the acknowledgement transition; the replay branch above
+        // returns before reaching this point.
+        await incrementCoreMailStatCounter(
+          this.models,
+          statDescriptor,
+          requestArg.outcome === 'processed'
+            ? 'acknowledgedProcessed'
+            : 'acknowledgedDiscarded',
+          updatedAt,
+          transactionArg,
+        );
         return { replayed: false };
       },
     );

package/ts/classes.maintenance.ts

@@ -2,6 +2,7 @@ import * as plugins from './plugins.js';
 import type { ICoreMailModels } from './classes.models.js';
 import type { CoreMailStorage } from './classes.storage.js';
 import { logCoreMailFailure } from './coremail.log.js';
+import { coreMailStatRetentionMs } from './coremail.stats.js';
 
 const sweepIntervalMs = 60_000;
 const batchSize = 100;
@@ -274,6 +275,19 @@ export class CoreMailMaintenanceService {
       }
       if (handles.length < batchSize) break;
     }
+    const statCutoff = now - coreMailStatRetentionMs;
+    while (!signalArg?.aborted) {
+      const statCounters = await this.models.StatCounter.exact.findStored({
+        filter: { dayUtc: { $lte: statCutoff } },
+        sort: { dayUtc: 1, statId: 1 },
+        limit: batchSize,
+      });
+      for (const statCounter of statCounters) {
+        signalArg?.throwIfAborted();
+        await this.models.StatCounter.exact.delete({ current: statCounter });
+      }
+      if (statCounters.length < batchSize) break;
+    }
     while (!signalArg?.aborted) {
       const counters = await this.models.QuotaCounter.exact.findStored({
         filter: {

package/ts/classes.models.ts

@@ -1,11 +1,16 @@
 import * as plugins from './plugins.js';
-import type { ISubmissionPartRecord, TTransferGrantPurpose } from './interfaces.js';
+import type {
+  ISubmissionPartRecord,
+  TCoreMailSubmissionSource,
+  TTransferGrantPurpose,
+} from './interfaces.js';
 import {
   assertCoreMailDesiredSnapshotRecord,
   assertCoreMailInboundDeliveryRecord,
   assertCoreMailInboundHandoffRecord,
   assertCoreMailQuotaCounterRecord,
   assertCoreMailRecipientHandleRecord,
+  assertCoreMailStatCounterRecord,
   assertCoreMailStateRecord,
   assertCoreMailSubmissionRecord,
   assertCoreMailTransferGrantRecord,
@@ -14,6 +19,7 @@ import {
   type ICoreMailInboundHandoffRecord,
   type ICoreMailQuotaCounterRecord,
   type ICoreMailRecipientHandleRecord,
+  type ICoreMailStatCounterRecord,
   type ICoreMailStateRecord,
   type ICoreMailSubmissionRecord,
   type ICoreMailTransferGrantRecord,
@@ -111,6 +117,33 @@ export class CoreMailTransferGrantDoc extends plugins.smartdata.SmartDataDbDoc<
   public consumedAt?: number;
 }
 
+@plugins.smartdata.exactPersistence({
+  assertDocument: assertCoreMailStatCounterRecord,
+  reconcileBy: 'statId',
+})
+export class CoreMailStatCounterDoc extends plugins.smartdata.SmartDataDbDoc<
+  CoreMailStatCounterDoc,
+  ICoreMailStatCounterRecord
+> implements ICoreMailStatCounterRecord {
+  declare public static exact: plugins.smartdata.TExact<CoreMailStatCounterDoc>;
+  public statId!: TCoreMailSha256;
+  public tenantId!: string;
+  public serviceId!: string;
+  public bindingId!: string;
+  public dayUtc!: number;
+  public submittedApi!: number;
+  public submittedSmtp!: number;
+  public delivered!: number;
+  public deferred!: number;
+  public failed!: number;
+  public deadLettered!: number;
+  public received!: number;
+  public acknowledgedProcessed!: number;
+  public acknowledgedDiscarded!: number;
+  public createdAt!: number;
+  public updatedAt!: number;
+}
+
 @plugins.smartdata.exactPersistence({
   assertDocument: assertCoreMailSubmissionRecord,
   reconcileBy: 'submissionId',
@@ -129,6 +162,8 @@ export class CoreMailSubmissionDoc extends plugins.smartdata.SmartDataDbDoc<
   public idempotencyKey!: string;
   public submissionDigest!: TCoreMailSha256;
   public state!: plugins.serveZoneInterfaces.data.TCoreMailSubmissionState;
+  public source?: TCoreMailSubmissionSource;
+  public statsSubmittedCountedAt?: number;
   public message!: plugins.serveZoneInterfaces.data.ICoreMailOutboundMessageDescriptor;
   public parts!: ISubmissionPartRecord[];
   public mimeObjectKey?: string;
@@ -237,6 +272,7 @@ export interface ICoreMailModels {
   State: typeof CoreMailStateDoc;
   DesiredSnapshot: typeof CoreMailDesiredSnapshotDoc;
   QuotaCounter: typeof CoreMailQuotaCounterDoc;
+  StatCounter: typeof CoreMailStatCounterDoc;
   TransferGrant: typeof CoreMailTransferGrantDoc;
   Submission: typeof CoreMailSubmissionDoc;
   InboundDelivery: typeof CoreMailInboundDeliveryDoc;
@@ -310,6 +346,34 @@ export const createCoreMailModels = (
         key: { state: 1, updatedAt: 1 },
       },
     ]) as typeof CoreMailDesiredSnapshotDoc,
+    StatCounter: bind(CoreMailStatCounterDoc, 'CoreMailStatCounter', [
+      'statId',
+      'tenantId',
+      'serviceId',
+      'bindingId',
+      'dayUtc',
+      'submittedApi',
+      'submittedSmtp',
+      'delivered',
+      'deferred',
+      'failed',
+      'deadLettered',
+      'received',
+      'acknowledgedProcessed',
+      'acknowledgedDiscarded',
+      'createdAt',
+      'updatedAt',
+    ], 'statId', [
+      {
+        name: 'authority_day_unique',
+        key: { tenantId: 1, serviceId: 1, bindingId: 1, dayUtc: 1 },
+        options: { unique: true },
+      },
+      {
+        name: 'day_retention',
+        key: { dayUtc: 1, statId: 1 },
+      },
+    ]) as typeof CoreMailStatCounterDoc,
     QuotaCounter: bind(CoreMailQuotaCounterDoc, 'CoreMailQuotaCounter', [
       'counterId',
       'tenantId',
@@ -375,6 +439,8 @@ export const createCoreMailModels = (
       'idempotencyKey',
       'submissionDigest',
       'state',
+      'source',
+      'statsSubmittedCountedAt',
       'message',
       'parts',
       'mimeObjectKey',

package/ts/classes.server.ts

@@ -10,6 +10,8 @@ import {
 import type { CoreMailDesiredStateService } from './classes.desiredstate.js';
 import type { CoreMailGateway } from './classes.gateway.js';
 import type { CoreMailInboundService } from './classes.inbound.js';
+import type { ICoreMailModels } from './classes.models.js';
+import type { CoreMailSmtpSubmissionServer } from './classes.smtpsubmission.js';
 import type { CoreMailStorage } from './classes.storage.js';
 import type { CoreMailSubmissionService } from './classes.submissions.js';
 import type { CoreMailTransferService } from './classes.transfer.js';
@@ -19,72 +21,13 @@ import type {
   ICoreMailReadiness,
   ICoreMailWorkloadSession,
 } from './interfaces.js';
-import { CoreMailQuotaExceededError } from './coremail.quota.js';
-import { logCoreMailFailure } from './coremail.log.js';
-
-type TErrorCode = plugins.serveZoneInterfaces.data.TCoreMailErrorCode;
+import { addPrivacySafeTypedHandler } from './coremail.errors.js';
+import { listCoreMailServiceMailStatistics } from './coremail.stats.js';
 
 interface ICoreMailTypedTools {
   localData: Record<string, unknown>;
 }
 
-export const privacySafeError = (
-  errorArg: unknown,
-): plugins.typedrequest.TypedResponseError => {
-  let code: TErrorCode = 'INVALID_REQUEST';
-  let retryable = false;
-  if (errorArg instanceof CoreMailAuthenticationError) {
-    code = 'AUTHENTICATION_FAILED';
-  } else if (errorArg instanceof CoreMailCapabilityDeniedError) {
-    code = 'CAPABILITY_DENIED';
-  } else if (errorArg instanceof CoreMailAuthorizationError) {
-    code = 'AUTHORITY_REVOKED';
-  } else if (errorArg instanceof CoreMailQuotaExceededError) {
-    return new plugins.typedrequest.TypedResponseError('CoreMail request rejected.', {
-      code: 'QUOTA_EXCEEDED',
-      retryable: true,
-      ...(errorArg.retryAfterMs === undefined
-        ? {}
-        : { retryAfterMs: errorArg.retryAfterMs }),
-    } satisfies plugins.serveZoneInterfaces.data.ICoreMailErrorData);
-  } else if (
-    errorArg instanceof plugins.serveZoneInterfaces.data.CoreMailContractError
-  ) {
-    code = errorArg.code;
-  } else if (
-    errorArg instanceof Error
-    && errorArg.message.includes('gateway')
-    && errorArg.message.includes('unavailable')
-  ) {
-    code = 'GATEWAY_UNAVAILABLE';
-    retryable = true;
-  } else if (
-    errorArg instanceof Error
-    && errorArg.message.includes('idempotency')
-  ) {
-    code = 'IDEMPOTENCY_CONFLICT';
-  } else if (
-    errorArg instanceof Error
-    && errorArg.message.includes('fence')
-  ) {
-    code = 'STATE_CONFLICT';
-    retryable = true;
-  } else if (
-    errorArg instanceof Error
-    && (
-      errorArg.message.includes('byte budget')
-      || errorArg.message.includes('content budget')
-      || errorArg.message.includes('exceeds')
-    )
-  ) {
-    code = 'PAYLOAD_LIMIT_EXCEEDED';
-  }
-  return new plugins.typedrequest.TypedResponseError('CoreMail request rejected.', {
-    code,
-    retryable,
-  } satisfies plugins.serveZoneInterfaces.data.ICoreMailErrorData);
-};
-
 const asPeer = (
   toolsArg: ICoreMailTypedTools | undefined,
 ): ICoreMailPeer => {
@@ -146,6 +89,7 @@ export class CoreMailServer {
   public constructor(
     private readonly config: ICoreMailConfig,
     private readonly database: plugins.smartdata.SmartdataDb,
+    private readonly models: ICoreMailModels,
     private readonly storage: CoreMailStorage,
     private readonly transfers: CoreMailTransferService,
     private readonly desiredState: CoreMailDesiredStateService,
@@ -153,6 +97,7 @@ export class CoreMailServer {
     private readonly submissions: CoreMailSubmissionService,
     private readonly inbound: CoreMailInboundService,
     private readonly gateway: CoreMailGateway,
+    private readonly smtpSubmission: CoreMailSmtpSubmissionServer,
     private readonly now: () => number = Date.now,
   ) {
     this.registerHandlers();
@@ -295,6 +240,10 @@ export class CoreMailServer {
       exactPurge: objectStorage,
       controlBootstrap: true,
       gateway: Boolean(desiredState) && this.gateway.isReady(),
+      // An absent or disabled listener is ready by omission; one the desired
+      // state asks for is ready only while it is actually accepting.
+      smtpSubmission: !desiredState?.smtp?.enabled
+        || this.smtpSubmission.isReady(),
     };
   }
 
@@ -343,6 +292,12 @@ export class CoreMailServer {
       this.auth.requireControlSession(asPeer(toolsArg));
       return { status: await this.desiredState.getStatus(requestArg) };
     });
+    this.addHandler<
+      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGetServiceMailStatistics
+    >('coreMailGetServiceMailStatistics', async (requestArg, toolsArg) => {
+      this.auth.requireControlSession(asPeer(toolsArg));
+      return await listCoreMailServiceMailStatistics(this.models, requestArg);
+    });
     this.addHandler<
       plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailPrepareOutboundSubmission
     >('coreMailPrepareOutboundSubmission', async (requestArg, toolsArg) => {
@@ -426,37 +381,18 @@ export class CoreMailServer {
     });
   }
 
-  /**
-   * Register one typed handler with uniform failure redaction.
-   *
-   * `handlerArg` is typed as typedrequest's own `THandlerFunction` rather than
-   * a hand-written signature: since typedrequest 8 both sides are wrapped in
-   * the `TReverseVirtualStreamDirections` mapped type, which does not reduce
-   * inside a generic wrapper, so any restatement of the shape stops being
-   * assignable. Concrete `IReq_*` types reduce normally, which is why the call
-   * sites below need no change.
-   */
+  /** Register one typed workload handler with uniform failure redaction. */
   private addHandler<
     TRequest extends plugins.typedrequestInterfaces.ITypedRequest,
   >(
     methodArg: TRequest['method'],
     handlerArg: plugins.typedrequest.THandlerFunction<TRequest>,
   ): void {
-    this.router.addTypedHandler(new plugins.typedrequest.TypedHandler<TRequest>(
+    addPrivacySafeTypedHandler<TRequest>(
+      this.router,
+      'server',
       methodArg,
-      async (requestArg, toolsArg) => {
-        try {
-          return await handlerArg(requestArg, toolsArg);
-        } catch (error) {
-          const safeError = privacySafeError(error);
-          if (safeError.errorData?.code === 'INVALID_REQUEST') {
-            await logCoreMailFailure('server', 'UNEXPECTED_HANDLER_FAILURE', error, {
-              method: methodArg,
-            }).catch(() => undefined);
-          }
-          throw safeError;
-        }
-      },
-    ));
+      handlerArg,
+    );
   }
 }