@serve.zone/coremail 1.0.0

package/ts/classes.gateway.ts

@@ -0,0 +1,811 @@
+import * as plugins from './plugins.js';
+import { type ICoreMailModels } from './classes.models.js';
+import { CoreMailTransferService } from './classes.transfer.js';
+import type { CoreMailInboundService } from './classes.inbound.js';
+import type {
+  ICoreMailConfig,
+  ITransferAuthority,
+} from './interfaces.js';
+import { logCoreMailFailure } from './coremail.log.js';
+import type { ICoreMailSubmissionRecord } from './coremail.persistence.js';
+import {
+  requireCoreMailIdentifier,
+  requireCoreMailSafeInteger,
+  requireCoreMailUuid,
+} from './coremail.selectors.js';
+
+type TDesiredState = plugins.serveZoneInterfaces.data.ICoreMailDesiredState;
+type TGatewayStatus = plugins.serveZoneInterfaces.data.ICoreMailGatewayOutboundStatus;
+type TGatewayDesired = plugins.serveZoneInterfaces.data.ICoreMailGatewayDesiredState;
+type TStoredSubmission =
+  plugins.smartdata.TStoredDocument<ICoreMailSubmissionRecord>;
+
+const gatewayAuthority = (serviceIdArg: string): ITransferAuthority => ({
+  kind: 'gateway',
+  tenantId: '-',
+  serviceId: serviceIdArg,
+  bindingId: '-',
+});
+
+const normalizeTransferOrigin = (valueArg: unknown): string => {
+  if (typeof valueArg !== 'string') {
+    throw new Error('CoreMail gateway returned an invalid transfer origin.');
+  }
+  const url = new URL(valueArg);
+  if (
+    url.protocol !== 'https:'
+    || url.username
+    || url.password
+    || url.search
+    || url.hash
+    || url.pathname !== '/'
+    || url.origin !== valueArg
+  ) {
+    throw new Error('CoreMail gateway returned a non-canonical transfer origin.');
+  }
+  return valueArg;
+};
+
+export const requireMatchingCoreMailTransferOrigin = (
+  valueArg: unknown,
+  expectedArg: string,
+): string => {
+  const transferOrigin = normalizeTransferOrigin(valueArg);
+  if (transferOrigin !== expectedArg) {
+    throw new Error('CoreMail gateway transfer origin does not match desired state.');
+  }
+  return transferOrigin;
+};
+
+const normalizeGatewayStatus = (valueArg: unknown): TGatewayStatus => {
+  try {
+    return plugins.serveZoneInterfaces.data
+      .normalizeCoreMailGatewayOutboundStatus(valueArg);
+  } catch {
+    throw new Error('CoreMail gateway returned an invalid outbound status.');
+  }
+};
+
+const waitForAbortableDelay = async (
+  delayMsArg: number,
+  signalArg: AbortSignal,
+): Promise<void> => {
+  if (signalArg.aborted) return;
+  await new Promise<void>((resolveArg) => {
+    const timeout = setTimeout(done, delayMsArg);
+    const onAbort = () => done();
+    function done() {
+      clearTimeout(timeout);
+      signalArg.removeEventListener('abort', onAbort);
+      resolveArg();
+    }
+    signalArg.addEventListener('abort', onAbort, { once: true });
+  });
+};
+
+export class CoreMailGateway {
+  private socket: plugins.typedsocket.TypedSocket | null = null;
+  private socketSignature = '';
+  private authenticatedSignature = '';
+  private transferOrigin: string | null = null;
+  private readonly stopController = new AbortController();
+  private reconcileTask: Promise<void> | null = null;
+  private statusSubscription: { unsubscribe(): void } | null = null;
+  private authenticationTask: Promise<void> = Promise.resolve();
+  private authenticatingSignature = '';
+
+  public constructor(
+    private readonly config: ICoreMailConfig,
+    private readonly models: ICoreMailModels,
+    private readonly transfers: CoreMailTransferService,
+    private readonly inbound: CoreMailInboundService,
+    private readonly getDesiredState: () => Promise<TDesiredState | null>,
+    private readonly now: () => number = Date.now,
+  ) {}
+
+  public start(): void {
+    if (this.reconcileTask) {
+      return;
+    }
+    this.reconcileTask = this.runReconciliationLoop();
+  }
+
+  public async stop(): Promise<void> {
+    this.stopController.abort();
+    await this.disconnect();
+    await this.reconcileTask?.catch(() => undefined);
+    this.reconcileTask = null;
+  }
+
+  public isReady(): boolean {
+    return Boolean(
+      this.socket
+      && this.socket.getStatus() === 'connected'
+      && this.authenticatedSignature === this.socketSignature
+      && this.transferOrigin,
+    );
+  }
+
+  public getTransferOrigin(): string | null {
+    return this.transferOrigin;
+  }
+
+  private async runReconciliationLoop(): Promise<void> {
+    while (!this.stopController.signal.aborted) {
+      try {
+        const desired = await this.getDesiredState();
+        const gateway = desired?.gateway;
+        if (!gateway) {
+          await this.disconnect();
+        } else {
+          const signature = JSON.stringify(gateway);
+          if (
+            !this.socket
+            || signature !== this.socketSignature
+            || this.socket.getStatus() !== 'connected'
+          ) {
+            await this.connect(gateway, signature);
+          } else if (
+            this.authenticatedSignature !== signature
+            && this.authenticatingSignature !== signature
+          ) {
+            this.queueAuthentication(gateway, signature);
+          }
+        }
+      } catch {
+        await logCoreMailFailure('gateway', 'RECONCILIATION_FAILED')
+          .catch(() => undefined);
+        await this.disconnect();
+      }
+      await waitForAbortableDelay(1_000, this.stopController.signal);
+    }
+  }
+
+  private async connect(
+    gatewayArg: TGatewayDesired,
+    signatureArg: string,
+  ): Promise<void> {
+    await this.disconnect();
+    if (this.stopController.signal.aborted) return;
+    const router = new plugins.typedrequest.TypedRouter();
+    router.addTypedHandler(new plugins.typedrequest.TypedHandler<
+      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayResolveRecipients
+    >('coreMailGatewayResolveRecipients', async (requestArg) => {
+      this.requireAuthenticated(signatureArg);
+      return await this.inbound.resolveRecipients(requestArg);
+    }));
+    router.addTypedHandler(new plugins.typedrequest.TypedHandler<
+      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayPrepareInboundHandoff
+    >('coreMailGatewayPrepareInboundHandoff', async (requestArg) => {
+      this.requireAuthenticated(signatureArg);
+      return await this.inbound.prepareInboundHandoff(requestArg);
+    }));
+    router.addTypedHandler(new plugins.typedrequest.TypedHandler<
+      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayCompleteInboundHandoff
+    >('coreMailGatewayCompleteInboundHandoff', async (requestArg) => {
+      this.requireAuthenticated(signatureArg);
+      return await this.inbound.completeInboundHandoff(requestArg);
+    }));
+    router.addTypedHandler(new plugins.typedrequest.TypedHandler<
+      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayCompleteOutboundHandoff
+    >('coreMailGatewayCompleteOutboundHandoff', async (requestArg) => {
+      this.requireAuthenticated(signatureArg);
+      return await this.completeOutboundHandoff(requestArg);
+    }));
+    const socket = await plugins.typedsocket.TypedSocket.createClient(
+      router,
+      gatewayArg.endpointUrl,
+      {
+        autoReconnect: false,
+        maxRetries: 0,
+        initialBackoffMs: 500,
+        maxBackoffMs: 15_000,
+        abortSignal: this.stopController.signal,
+      },
+    );
+    if (this.stopController.signal.aborted) {
+      await socket.stop();
+      return;
+    }
+    this.socket = socket;
+    this.socketSignature = signatureArg;
+    this.statusSubscription = socket.statusSubject.subscribe((statusArg) => {
+      if (statusArg === 'connected') {
+        this.queueAuthentication(gatewayArg, signatureArg);
+      } else {
+        this.authenticatedSignature = '';
+        this.transferOrigin = null;
+      }
+    });
+    this.queueAuthentication(gatewayArg, signatureArg);
+  }
+
+  private queueAuthentication(
+    gatewayArg: TGatewayDesired,
+    signatureArg: string,
+  ): void {
+    this.authenticationTask = this.authenticationTask
+      .catch(() => undefined)
+      .then(async () => {
+        if (
+          this.stopController.signal.aborted
+          || !this.socket
+          || this.socketSignature !== signatureArg
+          || this.socket.getStatus() !== 'connected'
+        ) {
+          return;
+        }
+        this.authenticatingSignature = signatureArg;
+        const credentialSecret = this.config.resolveRuntimeSecret(
+          gatewayArg.credentialSecretKey,
+        );
+        const response = await this.socket.createTypedRequest<
+          plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayAuthenticate
+        >('coreMailGatewayAuthenticate', undefined, {
+          timeoutMs: 5_000,
+          abortSignal: this.stopController.signal,
+        }).fire({
+          credentialId: gatewayArg.credentialId,
+          credentialVersion: gatewayArg.credentialVersion,
+          credentialSecret,
+        });
+        if (
+          response.authenticated !== true
+          || response.credentialId !== gatewayArg.credentialId
+          || response.credentialVersion !== gatewayArg.credentialVersion
+          || this.socketSignature !== signatureArg
+        ) {
+          throw new Error('CoreMail gateway authentication result is invalid.');
+        }
+        const transferOrigin = requireMatchingCoreMailTransferOrigin(
+          response.coreMailTransferOrigin,
+          gatewayArg.coreMailTransferOrigin,
+        );
+        this.transferOrigin = transferOrigin;
+        this.authenticatedSignature = signatureArg;
+      })
+      .catch(async () => {
+        if (this.socketSignature === signatureArg) {
+          this.authenticatedSignature = '';
+          this.transferOrigin = null;
+        }
+        await logCoreMailFailure('gateway', 'AUTHENTICATION_FAILED')
+          .catch(() => undefined);
+      })
+      .finally(() => {
+        if (this.authenticatingSignature === signatureArg) {
+          this.authenticatingSignature = '';
+        }
+      });
+  }
+
+  private requireAuthenticated(signatureArg: string): void {
+    if (
+      this.authenticatedSignature !== signatureArg
+      || !this.transferOrigin
+      || this.socket?.getStatus() !== 'connected'
+    ) {
+      throw new Error('CoreMail gateway session is unauthenticated.');
+    }
+  }
+
+  private async disconnect(): Promise<void> {
+    this.statusSubscription?.unsubscribe();
+    this.statusSubscription = null;
+    const socket = this.socket;
+    this.socket = null;
+    this.socketSignature = '';
+    this.authenticatedSignature = '';
+    this.authenticatingSignature = '';
+    this.transferOrigin = null;
+    await socket?.stop().catch(() => undefined);
+    await this.authenticationTask.catch(() => undefined);
+  }
+
+  public async prepareOutbound(
+    submissionArg: ICoreMailSubmissionRecord,
+    grantArg: plugins.serveZoneInterfaces.data.ICoreMailDownloadGrant,
+  ): Promise<{
+    transportMessageId: string;
+    status: TGatewayStatus;
+    replayed: boolean;
+  }> {
+    if (!this.socket || !this.isReady()) {
+      throw new Error('CoreMail gateway is unavailable.');
+    }
+    if (
+      !submissionArg.mimeSha256
+      || submissionArg.mimeLengthBytes === undefined
+    ) {
+      throw new Error('CoreMail outbound MIME is unavailable.');
+    }
+    const response = await this.socket.createTypedRequest<
+      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayPrepareOutboundHandoff
+    >('coreMailGatewayPrepareOutboundHandoff', undefined, {
+      timeoutMs: 30_000,
+      abortSignal: this.stopController.signal,
+    }).fire({
+      transportSubmissionId: submissionArg.submissionId,
+      submissionDigest: submissionArg.submissionDigest,
+      message: {
+        envelope: {
+          mailFrom: submissionArg.message.sender.address,
+          rcptTo: [
+            ...submissionArg.message.recipients.to,
+            ...(submissionArg.message.recipients.cc ?? []),
+            ...(submissionArg.message.recipients.bcc ?? []),
+          ].map((recipientArg) => recipientArg.address),
+        },
+        rawMime: {
+          sha256: submissionArg.mimeSha256,
+          lengthBytes: submissionArg.mimeLengthBytes,
+          contentType: 'message/rfc822',
+        },
+      },
+      grant: grantArg,
+    });
+    const status = normalizeGatewayStatus(response.status);
+    if (
+      typeof response.transportMessageId !== 'string'
+      || response.transportMessageId !== status.transportMessageId
+      || response.replayed !== Boolean(response.replayed)
+    ) {
+      throw new Error('CoreMail gateway outbound handoff response is invalid.');
+    }
+    return {
+      transportMessageId: response.transportMessageId,
+      status,
+      replayed: response.replayed,
+    };
+  }
+
+  public async getOutboundStatus(
+    transportMessageIdArg: string,
+  ): Promise<TGatewayStatus> {
+    if (!this.socket || !this.isReady()) {
+      throw new Error('CoreMail gateway is unavailable.');
+    }
+    const transportMessageId = requireCoreMailIdentifier(
+      transportMessageIdArg,
+      'transportMessageId',
+    );
+    const response = await this.socket.createTypedRequest<
+      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayGetOutboundStatus
+    >('coreMailGatewayGetOutboundStatus', undefined, {
+      timeoutMs: 10_000,
+      abortSignal: this.stopController.signal,
+    }).fire({ transportMessageId });
+    const status = normalizeGatewayStatus(response.status);
+    if (status.transportMessageId !== transportMessageId) {
+      throw new Error('CoreMail gateway status authority is invalid.');
+    }
+    return status;
+  }
+
+  private async completeOutboundHandoff(
+    requestArg:
+      plugins.serveZoneInterfaces.requests.coremail.IReq_CoreMailGatewayCompleteOutboundHandoff['request'],
+  ): Promise<{
+    status: plugins.serveZoneInterfaces.data.ICoreMailGatewayOutboundStatus;
+    replayed: boolean;
+  }> {
+    const sha256 = plugins.serveZoneInterfaces.data.normalizeCoreMailSha256(
+      requestArg.sha256,
+    );
+    const grantId = requireCoreMailUuid(requestArg.grantId, 'grantId');
+    const transportMessageId = requireCoreMailIdentifier(
+      requestArg.transportMessageId,
+      'transportMessageId',
+    );
+    const lengthBytes = requireCoreMailSafeInteger(
+      requestArg.lengthBytes,
+      'lengthBytes',
+    );
+    const grant = await this.transfers.assertConsumed({
+      grantId,
+      authority: gatewayAuthority(this.config.replica.serviceId),
+      purpose: 'outbound-mime-download',
+      expectedSha256: sha256,
+      lengthBytes,
+    });
+    const submission = await this.models.Submission.exact.findStoredOne({
+      mimeObjectKey: grant.objectKey,
+      mimeSha256: sha256,
+      mimeLengthBytes: lengthBytes,
+    });
+    if (!submission || !submission.mimeSha256) {
+      throw new Error('CoreMail outbound handoff is unavailable.');
+    }
+    if (
+      submission.transportMessageId
+      && submission.transportMessageId !== transportMessageId
+    ) {
+      throw new Error('CoreMail outbound transport identity conflicts.');
+    }
+    const replayed = submission.transportMessageId === transportMessageId;
+    const terminalState =
+      submission.state === 'delivered'
+      || submission.state === 'failed'
+      || submission.state === 'deadLettered'
+        ? submission.state
+        : undefined;
+    const status: TGatewayStatus = {
+      transportMessageId,
+      state: terminalState ?? 'queued',
+      attempts: submission.attempts,
+      ...(submission.error === undefined ? {} : { error: submission.error }),
+      ...(submission.deliveredAt === undefined
+        ? {}
+        : { deliveredAt: submission.deliveredAt }),
+      ...(submission.terminalAt === undefined
+        ? {}
+        : { terminalAt: submission.terminalAt }),
+      updatedAt: terminalState
+        ? (submission.gatewayStatusUpdatedAt ?? submission.updatedAt)
+        : this.now(),
+    };
+    if (!terminalState) {
+      await this.persistStatus(submission, status);
+    }
+    return {
+      status,
+      replayed,
+    };
+  }
+
+  public async persistStatus(
+    initialSubmissionArg: TStoredSubmission,
+    statusArg: TGatewayStatus,
+    leaseTokenArg?: string,
+  ): Promise<TStoredSubmission> {
+    let submission = initialSubmissionArg;
+    for (let attempt = 0; attempt < 4; attempt++) {
+      if (
+        submission.transportMessageId
+        && submission.transportMessageId !== statusArg.transportMessageId
+      ) {
+        throw new Error('CoreMail outbound transport identity conflicts.');
+      }
+      if (
+        submission.state === 'delivered'
+        || submission.state === 'failed'
+        || submission.state === 'deadLettered'
+      ) {
+        return submission;
+      }
+      if (
+        submission.gatewayStatusUpdatedAt !== undefined
+        && statusArg.updatedAt < submission.gatewayStatusUpdatedAt
+      ) {
+        return submission;
+      }
+      if (leaseTokenArg && submission.leaseToken !== leaseTokenArg) {
+        throw new Error('CoreMail outbound worker lease fence changed.');
+      }
+      const updatedAt = Math.max(this.now(), statusArg.updatedAt);
+      const terminal = statusArg.state === 'delivered'
+        || statusArg.state === 'failed'
+        || statusArg.state === 'deadLettered';
+      const nextAttemptAt = statusArg.state === 'deferred'
+        ? statusArg.nextAttemptAt
+        : undefined;
+      const workerRetryAt = terminal || statusArg.state === 'deferred'
+        ? undefined
+        : this.now() + 5_000;
+      const acceptedAt = submission.acceptedAt ?? Math.max(
+        submission.createdAt,
+        Math.min(
+          updatedAt,
+          statusArg.terminalAt ?? statusArg.deliveredAt ?? statusArg.updatedAt,
+        ),
+      );
+      if (
+        (statusArg.terminalAt !== undefined && statusArg.terminalAt < acceptedAt)
+        || (statusArg.deliveredAt !== undefined && statusArg.deliveredAt < acceptedAt)
+      ) {
+        throw new Error('CoreMail gateway status chronology conflicts.');
+      }
+      const result = await this.models.Submission.exact.transition({
+        current: submission,
+        change: (modelArg) => {
+          modelArg.transportMessageId = statusArg.transportMessageId;
+          modelArg.acceptedAt = acceptedAt;
+          modelArg.state = statusArg.state;
+          modelArg.attempts = Math.max(
+            submission.attempts,
+            statusArg.attempts,
+          );
+          modelArg.gatewayStatusUpdatedAt = statusArg.updatedAt;
+          if (nextAttemptAt === undefined) {
+            delete modelArg.nextAttemptAt;
+          } else {
+            modelArg.nextAttemptAt = nextAttemptAt;
+          }
+          if (workerRetryAt === undefined) {
+            delete modelArg.workerRetryAt;
+          } else {
+            modelArg.workerRetryAt = workerRetryAt;
+          }
+          if (statusArg.state === 'delivered') {
+            modelArg.deliveredAt = statusArg.deliveredAt;
+          } else {
+            delete modelArg.deliveredAt;
+          }
+          if (statusArg.terminalAt === undefined) {
+            delete modelArg.terminalAt;
+          } else {
+            modelArg.terminalAt = statusArg.terminalAt;
+          }
+          if (statusArg.error === undefined) {
+            delete modelArg.error;
+          } else {
+            modelArg.error = statusArg.error;
+          }
+          modelArg.updatedAt = updatedAt;
+          modelArg.revision += 1;
+          delete modelArg.leaseOwnerTaskId;
+          delete modelArg.leaseToken;
+          delete modelArg.leaseExpiresAt;
+        },
+      });
+      if (result.status === 'transitioned') {
+        return result.document;
+      }
+      if (leaseTokenArg) {
+        throw new Error('CoreMail outbound worker lease fence changed.');
+      }
+      const reloaded = await this.models.Submission.exact.findStoredOne({
+        submissionId: submission.submissionId,
+        tenantId: submission.tenantId,
+        serviceId: submission.serviceId,
+        bindingId: submission.bindingId,
+        bindingRevision: submission.bindingRevision,
+      });
+      if (!reloaded) {
+        throw new Error('CoreMail outbound submission disappeared.');
+      }
+      submission = reloaded;
+    }
+    throw new Error('CoreMail outbound status update remained contended.');
+  }
+}
+
+export class CoreMailOutboundWorker {
+  private readonly stopController = new AbortController();
+  private workerTask: Promise<void> | null = null;
+
+  public constructor(
+    private readonly models: ICoreMailModels,
+    private readonly transfers: CoreMailTransferService,
+    private readonly gateway: CoreMailGateway,
+    private readonly replicaTaskId: string,
+    private readonly serviceId: string,
+    private readonly now: () => number = Date.now,
+  ) {}
+
+  public start(): void {
+    this.workerTask ??= this.run();
+  }
+
+  public async stop(): Promise<void> {
+    this.stopController.abort();
+    await this.workerTask?.catch(() => undefined);
+    this.workerTask = null;
+  }
+
+  private async run(): Promise<void> {
+    while (!this.stopController.signal.aborted) {
+      if (this.gateway.isReady()) {
+        await this.processBatch().catch(async () => {
+          await logCoreMailFailure('worker', 'BATCH_FAILED')
+            .catch(() => undefined);
+        });
+      }
+      await waitForAbortableDelay(500, this.stopController.signal);
+    }
+  }
+
+  private async processBatch(): Promise<void> {
+    const now = this.now();
+    const candidates = await this.models.Submission.exact.findStored({
+      filter: {
+        $or: [
+          {
+            state: { $in: ['ready', 'accepted', 'queued', 'delivering'] },
+            $or: [
+              {
+                workerRetryAt: { $lte: now },
+                mimePublishedAt: { $exists: true },
+              },
+              {
+                leaseExpiresAt: { $lte: now },
+                leaseOwnerTaskId: { $ne: 'submission-finalizer' },
+                mimePublishedAt: { $exists: true },
+              },
+            ],
+          },
+          {
+            state: 'deferred',
+            $or: [
+              { nextAttemptAt: { $lte: now } },
+              {
+                leaseExpiresAt: { $lte: now },
+                leaseOwnerTaskId: { $ne: 'submission-finalizer' },
+                mimePublishedAt: { $exists: true },
+              },
+            ],
+          },
+        ],
+      },
+      sort: { workerRetryAt: 1, nextAttemptAt: 1, submissionId: 1 },
+      limit: 8,
+    });
+    for (const candidate of candidates) {
+      if (this.stopController.signal.aborted || !this.gateway.isReady()) {
+        return;
+      }
+      const claimNow = this.now();
+      const retryAt = candidate.state === 'deferred'
+        ? candidate.nextAttemptAt
+        : candidate.workerRetryAt;
+      const expiredLease = candidate.leaseExpiresAt !== undefined
+        && candidate.leaseExpiresAt <= claimNow;
+      if (
+        (
+          (retryAt === undefined || retryAt > claimNow)
+          && !expiredLease
+        )
+        || (
+          candidate.leaseExpiresAt !== undefined
+          && candidate.leaseExpiresAt > claimNow
+        )
+      ) {
+        continue;
+      }
+      const leaseToken = plugins.nodeCrypto.randomUUID();
+      const leaseExpiresAt = claimNow + 60_000;
+      const claim = await this.models.Submission.exact.transition({
+        current: candidate,
+        change: (modelArg) => {
+          if (candidate.transportMessageId) {
+            modelArg.state = 'delivering';
+            delete modelArg.nextAttemptAt;
+            delete modelArg.error;
+          }
+          modelArg.leaseOwnerTaskId = this.replicaTaskId;
+          modelArg.leaseToken = leaseToken;
+          modelArg.leaseExpiresAt = leaseExpiresAt;
+          modelArg.updatedAt = this.now();
+          modelArg.revision += 1;
+          delete modelArg.workerRetryAt;
+          if (!candidate.transportMessageId) {
+            modelArg.attempts += 1;
+          }
+        },
+      });
+      if (claim.status !== 'transitioned') {
+        continue;
+      }
+      await this.processClaimed(claim.document, leaseToken);
+    }
+  }
+
+  private async processClaimed(
+    submissionArg: TStoredSubmission,
+    leaseTokenArg: string,
+  ): Promise<void> {
+    try {
+      if (submissionArg.transportMessageId) {
+        const status = await this.gateway.getOutboundStatus(
+          submissionArg.transportMessageId,
+        );
+        await this.gateway.persistStatus(submissionArg, status, leaseTokenArg);
+        return;
+      }
+      if (
+        !submissionArg.mimeObjectKey
+        || !submissionArg.mimeSha256
+        || submissionArg.mimeLengthBytes === undefined
+        || submissionArg.mimePublishedAt === undefined
+      ) {
+        throw new Error('CoreMail queued submission has no immutable MIME object.');
+      }
+      const grant = await this.transfers.issueGrant({
+        authority: gatewayAuthority(this.serviceId),
+        purpose: 'outbound-mime-download',
+        method: 'GET',
+        expectedSha256: submissionArg.mimeSha256,
+        lengthBytes: submissionArg.mimeLengthBytes,
+        contentType: 'message/rfc822',
+        objectKey: submissionArg.mimeObjectKey,
+      }) as plugins.serveZoneInterfaces.data.ICoreMailDownloadGrant;
+      if (
+        submissionArg.leaseToken !== leaseTokenArg
+      ) {
+        throw new Error('CoreMail outbound grant fence changed.');
+      }
+      const grantPersisted = await this.models.Submission.exact.transition({
+        current: submissionArg,
+        change: (modelArg) => {
+          modelArg.transportGrantId = grant.grantId;
+          modelArg.updatedAt = this.now();
+          modelArg.revision += 1;
+        },
+      });
+      if (grantPersisted.status !== 'transitioned') {
+        throw new Error('CoreMail outbound grant fence changed.');
+      }
+      const withGrant = grantPersisted.document;
+      const response = await this.gateway.prepareOutbound(withGrant, grant);
+      await this.gateway.persistStatus(
+        withGrant,
+        response.status,
+        leaseTokenArg,
+      );
+    } catch {
+      await logCoreMailFailure('worker', 'DELIVERY_ATTEMPT_FAILED')
+        .catch(() => undefined);
+      const delayMs = Math.min(
+        15 * 60 * 1000,
+        1_000 * (2 ** Math.min(submissionArg.attempts, 9)),
+      );
+      const current = await this.models.Submission.exact.findStoredOne({
+        submissionId: submissionArg.submissionId,
+        tenantId: submissionArg.tenantId,
+        serviceId: submissionArg.serviceId,
+        bindingId: submissionArg.bindingId,
+        bindingRevision: submissionArg.bindingRevision,
+        leaseToken: leaseTokenArg,
+      });
+      if (current) {
+        await this.models.Submission.exact.transition({
+          current,
+          change: (modelArg) => {
+            if (current.transportMessageId) {
+              modelArg.state = 'deferred';
+              modelArg.nextAttemptAt = this.now() + delayMs;
+              modelArg.error = {
+                code: 'GATEWAY_UNAVAILABLE',
+                retryable: true,
+                retryAfterMs: delayMs,
+              };
+              delete modelArg.workerRetryAt;
+            } else if (
+              !current.mimeObjectKey
+              || !current.mimeSha256
+              || current.mimeLengthBytes === undefined
+              || current.mimePublishedAt === undefined
+            ) {
+              const terminalAt = this.now();
+              modelArg.state = 'failed';
+              modelArg.error = {
+                code: 'OBJECT_INTEGRITY_CONFLICT',
+                retryable: false,
+              };
+              modelArg.terminalAt = terminalAt;
+              modelArg.updatedAt = terminalAt;
+              delete modelArg.nextAttemptAt;
+              delete modelArg.workerRetryAt;
+            } else {
+              modelArg.state = 'ready';
+              modelArg.workerRetryAt = this.now() + delayMs;
+              delete modelArg.nextAttemptAt;
+              delete modelArg.error;
+            }
+            if (modelArg.state !== 'failed') {
+              delete modelArg.terminalAt;
+            }
+            delete modelArg.deliveredAt;
+            if (modelArg.state !== 'failed') {
+              modelArg.updatedAt = this.now();
+            }
+            modelArg.revision += 1;
+            delete modelArg.leaseOwnerTaskId;
+            delete modelArg.leaseToken;
+            delete modelArg.leaseExpiresAt;
+          },
+        }).catch(() => undefined);
+      }
+    }
+  }
+}