@serve.zone/coremail 1.0.0

package/ts/classes.maintenance.ts

@@ -0,0 +1,335 @@
+import * as plugins from './plugins.js';
+import type { ICoreMailModels } from './classes.models.js';
+import type { CoreMailStorage } from './classes.storage.js';
+import { logCoreMailFailure } from './coremail.log.js';
+
+const sweepIntervalMs = 60_000;
+const batchSize = 100;
+const transferRetirementGraceMs =
+  plugins.serveZoneInterfaces.data.coreMailLimits.transferGrantTtlMs
+  + plugins.serveZoneInterfaces.data.coreMailLimits.transferOverallTimeoutMs
+  + 2 * 60_000;
+
+export class CoreMailMaintenanceService {
+  private sweepTimer: ReturnType<typeof setInterval> | null = null;
+  private sweepTask: Promise<void> | null = null;
+  private sweepController: AbortController | null = null;
+
+  public constructor(
+    private readonly models: ICoreMailModels,
+    private readonly storage: CoreMailStorage,
+    private readonly now: () => number = Date.now,
+  ) {}
+
+  public start(): void {
+    if (this.sweepTimer) return;
+    this.sweepTimer = setInterval(() => {
+      void this.runSweep();
+    }, sweepIntervalMs);
+    this.sweepTimer.unref?.();
+  }
+
+  public async stop(): Promise<void> {
+    if (this.sweepTimer) {
+      clearInterval(this.sweepTimer);
+      this.sweepTimer = null;
+    }
+    this.sweepController?.abort();
+    await this.sweepTask?.catch(() => undefined);
+  }
+
+  public async sweep(signalArg?: AbortSignal): Promise<void> {
+    const now = this.now();
+    const terminalOutboundCutoff = now
+      - plugins.serveZoneInterfaces.data.coreMailRetentionPolicy.terminalOutboundMs;
+    const acknowledgedInboundCutoff = now
+      - plugins.serveZoneInterfaces.data.coreMailRetentionPolicy.acknowledgedInboundMs;
+    const idempotencyReceiptCutoff = now
+      - plugins.serveZoneInterfaces.data.coreMailRetentionPolicy.idempotencyReceiptMs;
+    const expiredCapabilityCutoff = now
+      - plugins.serveZoneInterfaces.data.coreMailRetentionPolicy.expiredCapabilityMs;
+    const transferRetirementCutoff = now - transferRetirementGraceMs;
+    while (!signalArg?.aborted) {
+      const terminalSubmissions = await this.models.Submission.exact.findStored({
+        filter: {
+          state: { $in: ['delivered', 'failed', 'deadLettered'] },
+          terminalAt: { $lte: terminalOutboundCutoff },
+          $or: [
+            { retiringAt: { $exists: false } },
+            { retiringAt: { $lte: transferRetirementCutoff } },
+          ],
+        },
+        sort: { terminalAt: 1, submissionId: 1 },
+        limit: batchSize,
+      });
+      for (const submission of terminalSubmissions) {
+        signalArg?.throwIfAborted();
+        await this.purgeSubmission(submission, signalArg);
+      }
+      if (terminalSubmissions.length < batchSize) break;
+    }
+    while (!signalArg?.aborted) {
+      const abandonedSubmissions = await this.models.Submission.exact.findStored({
+        filter: {
+          state: { $in: ['preparing', 'uploading', 'ready'] },
+          updatedAt: { $lte: idempotencyReceiptCutoff },
+          $or: [
+            { leaseExpiresAt: { $exists: false } },
+            { leaseExpiresAt: { $lte: now } },
+          ],
+        },
+        sort: { createdAt: 1, submissionId: 1 },
+        limit: batchSize,
+      });
+      for (const submission of abandonedSubmissions) {
+        signalArg?.throwIfAborted();
+        const retirement = await this.models.Submission.exact.transition({
+          current: submission,
+          change: (modelArg) => {
+            modelArg.state = 'failed';
+            modelArg.error = { code: 'STATE_CONFLICT', retryable: false };
+            modelArg.terminalAt = submission.updatedAt;
+            modelArg.retiringAt = now;
+            modelArg.updatedAt = now;
+            modelArg.revision += 1;
+            delete modelArg.nextAttemptAt;
+            delete modelArg.workerRetryAt;
+            delete modelArg.deliveredAt;
+            delete modelArg.leaseOwnerTaskId;
+            delete modelArg.leaseToken;
+            delete modelArg.leaseExpiresAt;
+          },
+        });
+        if (retirement.status !== 'transitioned') continue;
+      }
+      if (abandonedSubmissions.length < batchSize) break;
+    }
+    while (!signalArg?.aborted) {
+      const deliveries = await this.models.InboundDelivery.exact.findStored({
+        filter: {
+          state: 'acknowledged',
+          updatedAt: { $lte: acknowledgedInboundCutoff },
+        },
+        sort: { updatedAt: 1, deliveryId: 1 },
+        limit: batchSize,
+      });
+      for (const delivery of deliveries) {
+        signalArg?.throwIfAborted();
+        const retainedReferences = await this.models.InboundDelivery.exact.count({
+          rawMimeObjectKey: delivery.rawMimeObjectKey,
+          deliveryId: { $ne: delivery.deliveryId },
+          $or: [
+            { state: { $ne: 'acknowledged' } },
+            { updatedAt: { $gt: acknowledgedInboundCutoff } },
+          ],
+        });
+        if (retainedReferences === 0) {
+          await this.storage.purgeExact(delivery.rawMimeObjectKey, signalArg);
+        }
+        await this.models.InboundDelivery.exact.delete({ current: delivery });
+      }
+      if (deliveries.length < batchSize) break;
+    }
+    let handoffCursor: { createdAt: number; handoffId: string } | undefined;
+    while (!signalArg?.aborted) {
+      const incompleteHandoffs = await this.models.InboundHandoff.exact.findStored({
+        filter: {
+          $and: [
+            {
+              $or: [
+                {
+                  state: { $in: ['prepared', 'uploaded', 'failed'] },
+                  updatedAt: { $lte: idempotencyReceiptCutoff },
+                },
+                {
+                  state: 'retiring',
+                  retiringAt: { $lte: transferRetirementCutoff },
+                },
+              ],
+            },
+            ...(handoffCursor
+              ? [{
+                  $or: [
+                    { createdAt: { $gt: handoffCursor.createdAt } },
+                    {
+                      createdAt: handoffCursor.createdAt,
+                      handoffId: { $gt: handoffCursor.handoffId },
+                    },
+                  ],
+                }]
+              : []),
+          ],
+        },
+        sort: { createdAt: 1, handoffId: 1 },
+        limit: batchSize,
+      });
+      for (const handoff of incompleteHandoffs) {
+        signalArg?.throwIfAborted();
+        let retiringHandoff = handoff;
+        if (handoff.state !== 'retiring') {
+          const retirement = await this.models.InboundHandoff.exact.transition({
+            current: handoff,
+            change: (modelArg) => {
+              modelArg.state = 'retiring';
+              modelArg.retiringAt = now;
+              modelArg.updatedAt = now;
+              modelArg.revision += 1;
+            },
+          });
+          if (retirement.status !== 'transitioned') continue;
+          continue;
+        }
+        const deliveryReferences = await this.models.InboundDelivery.exact.count({
+          rawMimeObjectKey: retiringHandoff.objectKey,
+        });
+        if (deliveryReferences === 0) {
+          await this.storage.purgeExact(retiringHandoff.objectKey, signalArg);
+          await this.models.InboundHandoff.exact.delete({ current: retiringHandoff });
+        }
+      }
+      const lastHandoff = incompleteHandoffs.at(-1);
+      if (!lastHandoff || incompleteHandoffs.length < batchSize) break;
+      handoffCursor = {
+        createdAt: lastHandoff.createdAt,
+        handoffId: lastHandoff.handoffId,
+      };
+    }
+    while (!signalArg?.aborted) {
+      const completedHandoffs = await this.models.InboundHandoff.exact.findStored({
+        filter: {
+          state: 'completed',
+          completedAt: { $lte: idempotencyReceiptCutoff },
+        },
+        sort: { completedAt: 1, handoffId: 1 },
+        limit: batchSize,
+      });
+      for (const handoff of completedHandoffs) {
+        signalArg?.throwIfAborted();
+        await this.models.InboundHandoff.exact.delete({ current: handoff });
+      }
+      if (completedHandoffs.length < batchSize) break;
+    }
+    while (!signalArg?.aborted) {
+      const activeState = await this.models.State.exact.findStoredOne({
+        stateId: 'active',
+      });
+      const snapshots = await this.models.DesiredSnapshot.exact.findStored({
+        filter: {
+          $or: [
+            {
+              state: { $in: ['staged', 'superseded', 'failed'] },
+              updatedAt: { $lte: idempotencyReceiptCutoff },
+            },
+            {
+              state: 'retiring',
+              retiringAt: { $lte: transferRetirementCutoff },
+            },
+          ],
+          ...(activeState ? { snapshotId: { $ne: activeState.snapshotId } } : {}),
+        },
+        sort: { createdAt: 1, snapshotId: 1 },
+        limit: batchSize,
+      });
+      for (const snapshot of snapshots) {
+        signalArg?.throwIfAborted();
+        let retiringSnapshot = snapshot;
+        if (snapshot.state !== 'retiring') {
+          const retirement = await this.models.DesiredSnapshot.exact.transition({
+            current: snapshot,
+            change: (modelArg) => {
+              modelArg.state = 'retiring';
+              modelArg.retiringAt = now;
+              modelArg.updatedAt = now;
+            },
+          });
+          if (retirement.status !== 'transitioned') continue;
+          continue;
+        }
+        await this.storage.purgeExact(retiringSnapshot.objectKey, signalArg);
+        await this.models.DesiredSnapshot.exact.delete({ current: retiringSnapshot });
+      }
+      if (snapshots.length < batchSize) break;
+    }
+    while (!signalArg?.aborted) {
+      const grants = await this.models.TransferGrant.exact.findStored({
+        filter: { expiresAt: { $lte: expiredCapabilityCutoff } },
+        sort: { expiresAt: 1, grantId: 1 },
+        limit: batchSize,
+      });
+      for (const grant of grants) {
+        signalArg?.throwIfAborted();
+        await this.models.TransferGrant.exact.delete({ current: grant });
+      }
+      if (grants.length < batchSize) break;
+    }
+    while (!signalArg?.aborted) {
+      const handles = await this.models.RecipientHandle.exact.findStored({
+        filter: { expiresAt: { $lte: expiredCapabilityCutoff } },
+        sort: { expiresAt: 1, routingHandle: 1 },
+        limit: batchSize,
+      });
+      for (const handle of handles) {
+        signalArg?.throwIfAborted();
+        await this.models.RecipientHandle.exact.delete({ current: handle });
+      }
+      if (handles.length < batchSize) break;
+    }
+    while (!signalArg?.aborted) {
+      const counters = await this.models.QuotaCounter.exact.findStored({
+        filter: {
+          updatedAt: { $lte: idempotencyReceiptCutoff },
+          $or: [{
+            kind: { $in: ['outbound-minute', 'outbound-day'] },
+          }, {
+            kind: 'pending-inbound',
+            count: 0,
+          }],
+        },
+        sort: { updatedAt: 1, counterId: 1 },
+        limit: batchSize,
+      });
+      for (const counter of counters) {
+        signalArg?.throwIfAborted();
+        await this.models.QuotaCounter.exact.delete({ current: counter });
+      }
+      if (counters.length < batchSize) break;
+    }
+  }
+
+  private async purgeSubmission(
+    submissionArg: Awaited<ReturnType<
+      typeof this.models.Submission.exact.findStored
+    >>[number],
+    signalArg?: AbortSignal,
+  ): Promise<void> {
+    const objectKeys = new Set(submissionArg.parts.map((partArg) => partArg.objectKey));
+    if (submissionArg.mimeObjectKey) objectKeys.add(submissionArg.mimeObjectKey);
+    for (const objectKey of objectKeys) {
+      signalArg?.throwIfAborted();
+      await this.storage.purgeExact(objectKey, signalArg);
+    }
+    await this.models.Submission.exact.delete({ current: submissionArg });
+  }
+
+  private async runSweep(): Promise<void> {
+    if (this.sweepTask) return await this.sweepTask;
+    const controller = new AbortController();
+    this.sweepController = controller;
+    const task = this.sweep(controller.signal).catch(async () => {
+      if (!controller.signal.aborted) {
+        await logCoreMailFailure('retention', 'SWEEP_FAILED').catch(() => undefined);
+      }
+    });
+    this.sweepTask = task;
+    try {
+      await task;
+    } finally {
+      if (this.sweepTask === task) {
+        this.sweepTask = null;
+      }
+      if (this.sweepController === controller) {
+        this.sweepController = null;
+      }
+    }
+  }
+}