@serve.zone/coremail 1.2.0 → 1.3.0

package/ts/classes.smtpsubmission.ts

@@ -0,0 +1,466 @@
+import * as plugins from './plugins.js';
+import type { CoreMailAuth } from './classes.auth.js';
+import { stableStringify } from './classes.gateway.js';
+import type { CoreMailSubmissionService } from './classes.submissions.js';
+import { logCoreMailFailure } from './coremail.log.js';
+import { CoreMailQuotaExceededError } from './coremail.quota.js';
+import type {
+  ICoreMailConfig,
+  ICoreMailWorkloadSession,
+} from './interfaces.js';
+
+type TDesiredState = plugins.serveZoneInterfaces.data.ICoreMailDesiredState;
+type TSmtpDesired = plugins.serveZoneInterfaces.data.ICoreMailSmtpDesiredState;
+type TBinding = plugins.serveZoneInterfaces.data.ICoreMailBindingDesiredState;
+
+/** Session grants live no longer than smartmta's own hook-grant window. */
+const sessionTtlMs = 60 * 60 * 1000;
+/** Hard cap mirroring smartmta's hook-grant table. */
+const sessionMaximum = 4_096;
+const reconcileIntervalMs = 1_000;
+/** First retry delay after a failed start. */
+const startBackoffBaseMs = 1_000;
+/** Ceiling for the consecutive-failure backoff. */
+const startBackoffMaximumMs = 60_000;
+
+interface ICoreMailSmtpSessionGrant {
+  session: ICoreMailWorkloadSession;
+  binding: TBinding;
+  expiresAt: number;
+}
+
+const waitForAbortableDelay = async (
+  delayMsArg: number,
+  signalArg: AbortSignal,
+): Promise<void> => {
+  if (signalArg.aborted) return;
+  await new Promise<void>((resolveArg) => {
+    const timer = setTimeout(() => {
+      signalArg.removeEventListener('abort', onAbort);
+      resolveArg();
+    }, delayMsArg);
+    const onAbort = () => {
+      clearTimeout(timer);
+      resolveArg();
+    };
+    signalArg.addEventListener('abort', onAbort, { once: true });
+  });
+};
+
+/**
+ * CoreMail's SMTP submission listener (MSA).
+ *
+ * smartmta owns the transport only: it terminates TLS, runs SMTP AUTH over an
+ * encrypted channel, and hands every accepted message to `onMessageData` as
+ * the exact received bytes. Nothing is stored, queued or forwarded by it —
+ * `onMessageData` returns without `continueProcessing`, so the message ends at
+ * `CoreMailSubmissionService.submitRaw` and CoreMail's own durable outbound
+ * pipeline. The `MemoryStorageManager` handed to smartmta is therefore
+ * expected to stay empty for the life of the process; a test asserts that.
+ *
+ * The listener fails closed. It runs only when the desired state enables it
+ * and both PEM secrets resolve, because smartmta's Rust listener refuses AUTH
+ * on a cleartext session (538 5.7.11) and a listener without certificate
+ * material would advertise neither STARTTLS nor AUTH.
+ */
+export class CoreMailSmtpSubmissionServer {
+  private server: InstanceType<
+    typeof plugins.smartmta.UnifiedEmailServer
+  > | null = null;
+  private storageManager: InstanceType<
+    typeof plugins.smartmta.MemoryStorageManager
+  > | null = null;
+  private serverSignature = '';
+  private readonly stopController = new AbortController();
+  private reconcileTask: Promise<void> | null = null;
+  private readonly sessions = new Map<string, ICoreMailSmtpSessionGrant>();
+  private unavailableReason: string | null = 'CoreMail SMTP listener has not started.';
+  private loggedUnavailableSignature = '';
+  /** Signature of the configuration the last start was attempted for. */
+  private attemptedSignature = '';
+  private startFailures = 0;
+  private reportedUnavailable = 0;
+  private nextStartAttemptAt = 0;
+
+  public constructor(
+    private readonly config: ICoreMailConfig,
+    private readonly auth: CoreMailAuth,
+    private readonly submissions: CoreMailSubmissionService,
+    private readonly getDesiredState: () => Promise<TDesiredState | null>,
+    private readonly now: () => number = Date.now,
+  ) {}
+
+  public start(): void {
+    this.reconcileTask ??= this.runReconciliationLoop();
+  }
+
+  public async stop(): Promise<void> {
+    this.stopController.abort();
+    await this.reconcileTask?.catch(() => undefined);
+    this.reconcileTask = null;
+    await this.shutdownServer();
+    this.sessions.clear();
+    this.resetStartBackoff('');
+  }
+
+  /** Whether the listener is currently accepting submissions. */
+  public isReady(): boolean {
+    return this.server !== null && this.unavailableReason === null;
+  }
+
+  /**
+   * Operator-facing view of the start-retry state. A listener that cannot bind
+   * (busy port, unusable key pair, missing mailer-bin) backs off instead of
+   * respawning the Rust bridge once per reconcile tick.
+   */
+  public getListenerDiagnostics(): {
+    startFailures: number;
+    reportedUnavailable: number;
+    nextStartAttemptAt: number;
+  } {
+    return {
+      startFailures: this.startFailures,
+      reportedUnavailable: this.reportedUnavailable,
+      nextStartAttemptAt: this.nextStartAttemptAt,
+    };
+  }
+
+  /** In-memory storage handed to smartmta, for the emptiness assertion. */
+  public getStorageManager(): InstanceType<
+    typeof plugins.smartmta.MemoryStorageManager
+  > | null {
+    return this.storageManager;
+  }
+
+  private async runReconciliationLoop(): Promise<void> {
+    while (!this.stopController.signal.aborted) {
+      try {
+        await this.reconcile();
+      } catch (error) {
+        await logCoreMailFailure('smtp', 'RECONCILIATION_FAILED', error)
+          .catch(() => undefined);
+      }
+      await waitForAbortableDelay(
+        reconcileIntervalMs,
+        this.stopController.signal,
+      );
+    }
+  }
+
+  private async reconcile(): Promise<void> {
+    const desired = await this.getDesiredState();
+    const smtp = desired?.smtp;
+    if (!smtp || !smtp.enabled) {
+      await this.markUnavailable(
+        smtp ? 'CoreMail SMTP listener is disabled.' : 'CoreMail SMTP listener is not configured.',
+        stableStringify(smtp ?? null),
+      );
+      return;
+    }
+    const material = this.resolveTlsMaterial(smtp);
+    if (!material) {
+      await this.markUnavailable(
+        'CoreMail SMTP listener TLS material does not resolve.',
+        stableStringify(smtp),
+      );
+      return;
+    }
+    const signature = stableStringify({ smtp, certPem: material.certPem });
+    if (this.server && signature === this.serverSignature) return;
+    if (signature === this.attemptedSignature) {
+      // The same configuration failed already; wait out its backoff rather
+      // than spawning and tearing down the Rust bridge every tick.
+      if (this.now() < this.nextStartAttemptAt) return;
+    } else {
+      // A changed smtp section always restarts immediately.
+      this.resetStartBackoff(signature);
+    }
+    await this.shutdownServer();
+    if (this.stopController.signal.aborted) return;
+    await this.startServer(smtp, material, signature);
+  }
+
+  private resetStartBackoff(signatureArg: string): void {
+    this.attemptedSignature = signatureArg;
+    this.startFailures = 0;
+    this.nextStartAttemptAt = 0;
+  }
+
+  /**
+   * Resolve both PEM secrets. A listener that cannot present a certificate
+   * must never come up: smartmta would then offer neither STARTTLS nor AUTH,
+   * and any client configured for submission would fail open into plaintext.
+   */
+  private resolveTlsMaterial(
+    smtpArg: TSmtpDesired,
+  ): { certPem: string; keyPem: string } | null {
+    try {
+      const certPem = this.config.resolveRuntimeSecret(
+        smtpArg.tls.certificatePemSecretKey,
+      );
+      const keyPem = this.config.resolveRuntimeSecret(
+        smtpArg.tls.privateKeyPemSecretKey,
+      );
+      if (
+        typeof certPem !== 'string'
+        || !certPem.includes('BEGIN')
+        || typeof keyPem !== 'string'
+        || !keyPem.includes('BEGIN')
+      ) {
+        return null;
+      }
+      return { certPem, keyPem };
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Record why the listener is not serving and log it once per distinct
+   * reason. Repeating the same reason every reconcile tick would drown the
+   * operator in noise without adding information.
+   */
+  private async reportUnavailable(
+    reasonArg: string,
+    signatureArg: string,
+    codeArg: 'LISTENER_UNAVAILABLE' | 'LISTENER_START_FAILED',
+    errorArg?: unknown,
+  ): Promise<void> {
+    await this.shutdownServer();
+    this.unavailableReason = reasonArg;
+    const logSignature = `${codeArg}:${signatureArg}`;
+    if (this.loggedUnavailableSignature === logSignature) return;
+    this.loggedUnavailableSignature = logSignature;
+    this.reportedUnavailable += 1;
+    await logCoreMailFailure('smtp', codeArg, errorArg, {
+      reason: reasonArg,
+    }).catch(() => undefined);
+  }
+
+  /**
+   * The configuration is not asking for a listener right now, so no start is
+   * pending and a later enable must take effect on its first tick.
+   */
+  private async markUnavailable(
+    reasonArg: string,
+    signatureArg: string,
+  ): Promise<void> {
+    this.resetStartBackoff('');
+    await this.reportUnavailable(reasonArg, signatureArg, 'LISTENER_UNAVAILABLE');
+  }
+
+  private async startServer(
+    smtpArg: TSmtpDesired,
+    materialArg: { certPem: string; keyPem: string },
+    signatureArg: string,
+  ): Promise<void> {
+    const storageManager = new plugins.smartmta.MemoryStorageManager();
+    const server = new plugins.smartmta.UnifiedEmailServer({ storageManager }, {
+      ports: [smtpArg.port],
+      hostname: smtpArg.hostname,
+      // CoreMail hosts no inbound domain of its own here: this listener only
+      // relays authenticated submissions to the durable outbound pipeline.
+      domains: [],
+      routes: [{
+        name: 'submission',
+        match: { authenticated: true },
+        action: { type: 'process', allowRelay: true },
+      }],
+      persistRoutes: false,
+      dkimKeyProvisioning: 'caller-managed',
+      tls: { certPem: materialArg.certPem, keyPem: materialArg.keyPem },
+      auth: { required: true },
+      queue: { storageMode: 'memory' },
+      smtp: {
+        maxConcurrentMessages: 16,
+        recipientValidation: false,
+        authenticationTimeoutMs: 4_000,
+        messageAcceptanceTimeoutMs: 30_000,
+      },
+      maxMessageSize:
+        plugins.serveZoneInterfaces.data.coreMailLimits.serializedMimeBytes,
+      hooks: {
+        onAuthenticate: (contextArg) => this.onAuthenticate(contextArg),
+        onMessageData: (contextArg) => this.onMessageData(contextArg),
+      },
+    });
+    try {
+      await server.start();
+    } catch (error) {
+      await server.stop().catch(() => undefined);
+      this.startFailures += 1;
+      this.nextStartAttemptAt = this.now() + Math.min(
+        startBackoffMaximumMs,
+        startBackoffBaseMs * 2 ** (this.startFailures - 1),
+      );
+      await this.reportUnavailable(
+        'CoreMail SMTP listener failed to start.',
+        signatureArg,
+        'LISTENER_START_FAILED',
+        error,
+      );
+      return;
+    }
+    this.server = server;
+    this.storageManager = storageManager;
+    this.serverSignature = signatureArg;
+    this.unavailableReason = null;
+    this.loggedUnavailableSignature = '';
+    this.startFailures = 0;
+    this.nextStartAttemptAt = 0;
+  }
+
+  private async shutdownServer(): Promise<void> {
+    const server = this.server;
+    this.server = null;
+    this.serverSignature = '';
+    this.sessions.clear();
+    if (!server) return;
+    await server.stop().catch(async (error) => {
+      await logCoreMailFailure('smtp', 'LISTENER_SHUTDOWN_FAILED', error)
+        .catch(() => undefined);
+    });
+  }
+
+  private pruneSessions(): void {
+    const now = this.now();
+    for (const [sessionId, grant] of this.sessions) {
+      if (grant.expiresAt <= now) this.sessions.delete(sessionId);
+    }
+    while (this.sessions.size > sessionMaximum) {
+      const oldest = this.sessions.keys().next();
+      if (oldest.done) break;
+      this.sessions.delete(oldest.value);
+    }
+  }
+
+  private async onAuthenticate(
+    contextArg: plugins.smartmta.ISmtpAuthenticateContext,
+  ): Promise<plugins.smartmta.ISmtpAuthenticateDecision> {
+    // The SMTP username is the binding id; the password is any accepted
+    // credential secret of that binding.
+    const authenticated = await this.auth.authenticateSmtpSubmission(
+      contextArg.username,
+      contextArg.password,
+    );
+    if (!authenticated) {
+      return { accepted: false, message: 'CoreMail binding credential rejected.' };
+    }
+    const { binding } = authenticated;
+    const session: ICoreMailWorkloadSession = {
+      kind: 'workload',
+      peerId: `smtp:${contextArg.sessionId}`,
+      tenantId: binding.tenantId,
+      serviceId: binding.serviceId,
+      bindingId: binding.bindingId,
+      bindingRevision: binding.revision,
+      configEpoch: authenticated.configEpoch,
+      credentialId: authenticated.credentialId,
+      credentialVersion: authenticated.credentialVersion,
+      bindingState: binding.state === 'draining' ? 'draining' : 'active',
+      capabilities: [...binding.capabilities],
+      allowedOperations: [
+        ...plugins.serveZoneInterfaces.data.resolveCoreMailWorkloadOperations(
+          binding.state,
+          binding.capabilities,
+        ),
+      ],
+      coreMailTransferOrigin: '-',
+      authenticatedAt: this.now(),
+    };
+    this.sessions.delete(contextArg.sessionId);
+    this.sessions.set(contextArg.sessionId, {
+      session,
+      binding,
+      expiresAt: this.now() + sessionTtlMs,
+    });
+    this.pruneSessions();
+    return {
+      accepted: true,
+      // smartmta enforces the same sender scope CoreMail enforces again in
+      // submitRaw, so an out-of-scope MAIL FROM is refused at RCPT time.
+      scope: { senders: [...binding.allowedSenders] },
+    };
+  }
+
+  private async onMessageData(
+    contextArg: plugins.smartmta.IMessageAcceptanceContext,
+  ): Promise<plugins.smartmta.IMessageAcceptanceDecision> {
+    const grant = this.sessions.get(contextArg.session.id);
+    if (!grant || grant.expiresAt <= this.now()) {
+      this.sessions.delete(contextArg.session.id);
+      return {
+        accepted: false,
+        smtpCode: 451,
+        smtpMessage: '4.7.1 Authentication scope unavailable; authenticate again',
+      };
+    }
+    try {
+      // smartmta strips the CRLF that preceded the DATA terminator; the
+      // durable raw queue requires canonical CRLF-terminated bytes.
+      const rawMime = new Uint8Array(contextArg.rawMessage.byteLength + 2);
+      rawMime.set(contextArg.rawMessage, 0);
+      rawMime.set([0x0d, 0x0a], contextArg.rawMessage.byteLength);
+      await this.submissions.submitRaw(grant.session, grant.binding, {
+        envelope: {
+          from: contextArg.session.envelope.mailFrom.address,
+          to: contextArg.session.envelope.rcptTo.map(
+            (recipientArg) => recipientArg.address,
+          ),
+        },
+        rawMime,
+      });
+      // No continueProcessing: smartmta stores, queues and forwards nothing.
+      return { accepted: true };
+    } catch (error) {
+      return this.refusalFor(error);
+    }
+  }
+
+  private refusalFor(
+    errorArg: unknown,
+  ): plugins.smartmta.IMessageAcceptanceDecision {
+    if (errorArg instanceof CoreMailQuotaExceededError) {
+      return {
+        accepted: false,
+        smtpCode: 452,
+        smtpMessage: '4.2.1 Mail quota exceeded; try again later',
+      };
+    }
+    const message = errorArg instanceof Error ? errorArg.message : '';
+    if (message.includes('sender is not authorized')) {
+      return {
+        accepted: false,
+        smtpCode: 553,
+        smtpMessage: '5.7.1 Sender address not permitted',
+      };
+    }
+    if (message.includes('byte budget')) {
+      return {
+        accepted: false,
+        smtpCode: 552,
+        smtpMessage: '5.3.4 Message exceeds its size limit',
+      };
+    }
+    if (
+      errorArg instanceof Error
+      && (
+        errorArg.name === 'CoreMailCapabilityDeniedError'
+        || errorArg.name === 'CoreMailAuthorizationError'
+      )
+    ) {
+      return {
+        accepted: false,
+        smtpCode: 550,
+        smtpMessage: '5.7.1 Submission is not permitted by this authority',
+      };
+    }
+    void logCoreMailFailure('smtp', 'SUBMISSION_FAILED', errorArg)
+      .catch(() => undefined);
+    return {
+      accepted: false,
+      smtpCode: 451,
+      smtpMessage: '4.3.0 Submission temporarily unavailable',
+    };
+  }
+}

package/ts/classes.storage.ts

@@ -372,6 +372,60 @@ export class CoreMailStorage {
     );
   }
 
+  /**
+   * Read at most `headBytesArg` leading bytes of a stored object.
+   *
+   * The object's integrity was already proven when its transfer grant was
+   * consumed, so a header scan does not need to re-hash the whole payload — it
+   * reads a bounded prefix and cancels. The verified full-object readers above
+   * remain the only way to obtain content that is acted on as a whole.
+   */
+  public async readObjectHeadBytes(
+    objectKeyArg: string,
+    lengthBytesArg: number,
+    sha256Arg: TCoreMailSha256,
+    maximumBytesArg: number,
+    headBytesArg: number,
+    signalArg?: AbortSignal,
+  ): Promise<Uint8Array> {
+    if (!Number.isSafeInteger(headBytesArg) || headBytesArg < 1) {
+      throw new Error('CoreMail object head read length is invalid.');
+    }
+    const verified = await this.getVerifiedStream(
+      objectKeyArg,
+      lengthBytesArg,
+      sha256Arg,
+      maximumBytesArg,
+      signalArg,
+    );
+    const reader = verified.stream.getReader();
+    const chunks: Uint8Array[] = [];
+    let total = 0;
+    try {
+      while (total < headBytesArg) {
+        const next = await reader.read();
+        if (next.done) break;
+        chunks.push(next.value);
+        total += next.value.byteLength;
+      }
+    } finally {
+      await reader.cancel().catch(() => undefined);
+      reader.releaseLock();
+      await verified.cancel().catch(() => undefined);
+      // The digest is deliberately not awaited: only a prefix was consumed.
+      verified.completion.catch(() => undefined);
+    }
+    const head = new Uint8Array(Math.min(total, headBytesArg));
+    let offset = 0;
+    for (const chunk of chunks) {
+      if (offset >= head.byteLength) break;
+      const slice = chunk.subarray(0, head.byteLength - offset);
+      head.set(slice, offset);
+      offset += slice.byteLength;
+    }
+    return head;
+  }
+
   public async getVerifiedBytes(
     objectKeyArg: string,
     lengthBytesArg: number,