@push.rocks/smartmta 8.1.0 → 8.1.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartmta",
-  "version": "8.1.0",
+  "version": "8.1.1",
   "description": "A high-performance, enterprise-grade Mail Transfer Agent (MTA) built from scratch in TypeScript with Rust acceleration.",
   "keywords": [
     "mta",

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartmta',
-  version: '8.1.0',
+  version: '8.1.1',
   description: 'A high-performance, enterprise-grade Mail Transfer Agent (MTA) built from scratch in TypeScript with Rust acceleration.'
 }

package/ts/index.ts

@@ -13,4 +13,5 @@ export const smartMtaCapabilities = Object.freeze({
   outboundAttachments: true,
   topologyAwareSelfMx: true,
   durableDeliveryFailureOutbox: true,
+  exactAcceptedEnvelopeDispatch: true,
 } as const);

package/ts/mail/delivery/classes.delivery.queue.ts

@@ -58,6 +58,10 @@ export interface IBounceOperation {
  */
 export interface IQueueItem {
   id: string;
+  /** Stable caller-supplied key used to make enqueue replay-safe. */
+  idempotencyKey?: string;
+  /** Fingerprint of the exact work associated with the idempotency key. */
+  idempotencyFingerprint?: string;
   processingMode: EmailProcessingMode;
   processingResult: Email;
   /** Materialized attachment count for queue/detail consumers. */
@@ -86,6 +90,8 @@ interface IStoredEmail {
 interface IStoredQueueItemV1 {
   schemaVersion: 1;
   id: string;
+  idempotencyKey?: string;
+  idempotencyFingerprint?: string;
   processingMode: EmailProcessingMode;
   processingResult: IStoredEmail;
   route?: IEmailRoute;
@@ -121,6 +127,17 @@ type IStoredQueueItem = IStoredQueueItemV1 | IStoredQueueItemV2;
 
 type TQueueLifecycleState = 'new' | 'initialized' | 'shutting-down' | 'closed';
 
+interface IStoredQueueIdempotencyTombstone {
+  schemaVersion: 1;
+  id: string;
+  idempotencyKey: string;
+  idempotencyFingerprint: string;
+  expiresAt: number;
+}
+
+export const ACCEPTED_ENVELOPE_IDEMPOTENCY_RETENTION_MS = 31 * 24 * 60 * 60 * 1000;
+const QUEUE_IDEMPOTENCY_TOMBSTONE_CLEANUP_LIMIT = 256;
+
 export interface IQueueFailureOptions {
   /** Unknown failures default to retryable so temporary internal faults do not lose mail. */
   retryable?: boolean;
@@ -159,6 +176,15 @@ export interface IQueueOptions {
   maxRetryDelay?: number;
 }
 
+export interface IQueueEnqueueOptions {
+  /** Stable key that resolves to one deterministic queue item id. */
+  idempotencyKey?: string;
+  /** Required with idempotencyKey to reject reuse for different work. */
+  idempotencyFingerprint?: string;
+  /** Fail closed unless the queue is backed by managed blob storage. */
+  requireDurable?: boolean;
+}
+
 /**
  * Queue statistics interface
  */
@@ -225,6 +251,11 @@ export class UnifiedDeliveryQueue extends EventEmitter {
   private totalProcessed: number = 0;
   private ownsManagedPrefix = false;
   private lifecycleState: TQueueLifecycleState = 'new';
+  private readonly idempotentEnqueueFlights = new Map<string, {
+    fingerprint: string;
+    promise: Promise<string>;
+  }>();
+  private idempotencyTombstoneCleanupCursor?: string;
   
   /**
    * Create a new unified delivery queue
@@ -279,6 +310,143 @@ export class UnifiedDeliveryQueue extends EventEmitter {
     }
   }
 
+  /** Whether queue entries are committed through caller-managed blob storage. */
+  public hasManagedStorage(): boolean {
+    return !!this.options.storageManager;
+  }
+
+  /** Whether managed storage can list a backend-bounded page of keys. */
+  public hasPaginatedManagedStorage(): boolean {
+    return typeof this.options.storageManager?.listPage === 'function';
+  }
+
+  private idempotentQueueId(idempotencyKey: string): string {
+    return `idempotent-${createHash('sha256')
+      .update('smartmta-queue-v1\0')
+      .update(idempotencyKey)
+      .digest('hex')}`;
+  }
+
+  private idempotencyTombstonePrefix(): string {
+    return `${this.options.storagePrefix.slice(0, -1)}.idempotency/`;
+  }
+
+  private idempotencyTombstoneKey(id: string): string {
+    return `${this.idempotencyTombstonePrefix()}${id}.json`;
+  }
+
+  private async readIdempotencyTombstone(
+    id: string,
+  ): Promise<IStoredQueueIdempotencyTombstone | null> {
+    const storageManager = this.options.storageManager;
+    if (!storageManager) return null;
+    const key = this.idempotencyTombstoneKey(id);
+    const value = await storageManager.get(key);
+    if (!value) return null;
+    const tombstone = this.parseIdempotencyTombstone(value, key, id);
+    if (tombstone.expiresAt <= Date.now()) {
+      await storageManager.delete(key);
+      return null;
+    }
+    return tombstone;
+  }
+
+  private parseIdempotencyTombstone(
+    value: Buffer,
+    key: string,
+    expectedId: string,
+  ): IStoredQueueIdempotencyTombstone {
+    let tombstone: IStoredQueueIdempotencyTombstone;
+    try {
+      tombstone = JSON.parse(value.toString('utf8')) as IStoredQueueIdempotencyTombstone;
+    } catch {
+      throw new Error(`Invalid queue idempotency tombstone at ${key}`);
+    }
+    if (
+      tombstone?.schemaVersion !== 1
+      || tombstone.id !== expectedId
+      || typeof tombstone.idempotencyKey !== 'string'
+      || !tombstone.idempotencyKey
+      || tombstone.idempotencyKey.length > 512
+      || this.idempotentQueueId(tombstone.idempotencyKey) !== expectedId
+      || typeof tombstone.idempotencyFingerprint !== 'string'
+      || !/^[a-f0-9]{64}$/.test(tombstone.idempotencyFingerprint)
+      || !Number.isFinite(tombstone.expiresAt)
+    ) {
+      throw new Error(`Invalid queue idempotency tombstone at ${key}`);
+    }
+    return tombstone;
+  }
+
+  private async persistIdempotencyTombstone(
+    id: string,
+    idempotencyKey: string,
+    idempotencyFingerprint: string,
+  ): Promise<void> {
+    const storageManager = this.options.storageManager;
+    if (!storageManager) return;
+    const tombstone: IStoredQueueIdempotencyTombstone = {
+      schemaVersion: 1,
+      id,
+      idempotencyKey,
+      idempotencyFingerprint,
+      expiresAt: Date.now() + ACCEPTED_ENVELOPE_IDEMPOTENCY_RETENTION_MS,
+    };
+    await storageManager.set(
+      this.idempotencyTombstoneKey(id),
+      Buffer.from(JSON.stringify(tombstone)),
+    );
+  }
+
+  private async cleanupExpiredIdempotencyTombstones(
+    limit = QUEUE_IDEMPOTENCY_TOMBSTONE_CLEANUP_LIMIT,
+  ): Promise<void> {
+    const storageManager = this.options.storageManager;
+    if (!storageManager?.listPage || limit < 1) return;
+    const prefix = this.idempotencyTombstonePrefix();
+    const page = await storageManager.listPage(prefix, {
+      limit,
+      ...(this.idempotencyTombstoneCleanupCursor
+        ? { continuationToken: this.idempotencyTombstoneCleanupCursor }
+        : {}),
+    });
+    if (
+      !page
+      || !Array.isArray(page.keys)
+      || page.keys.length > limit
+      || page.keys.some((key) => typeof key !== 'string' || !key.startsWith(prefix))
+      || new Set(page.keys).size !== page.keys.length
+      || (
+        page.continuationToken !== undefined
+        && (
+          typeof page.continuationToken !== 'string'
+          || page.continuationToken.length === 0
+          || page.continuationToken === this.idempotencyTombstoneCleanupCursor
+        )
+      )
+    ) {
+      throw new Error('Invalid paginated queue idempotency tombstone listing');
+    }
+    this.idempotencyTombstoneCleanupCursor = page.continuationToken;
+    const now = Date.now();
+    for (const key of page.keys) {
+      const value = await storageManager.get(key);
+      if (!value) continue;
+      try {
+        if (!key.endsWith('.json')) {
+          throw new Error(`Invalid queue idempotency tombstone key: ${key}`);
+        }
+        const id = key.slice(prefix.length, -'.json'.length);
+        const tombstone = this.parseIdempotencyTombstone(value, key, id);
+        if (tombstone.expiresAt <= now) {
+          await storageManager.delete(key);
+        }
+      } catch {
+        await storageManager.delete(key);
+      }
+    }
+  }
+
   private createInitialRecipientStates(email: Email): IRecipientDeliveryState[] {
     const byRecipient = new Map<string, IRecipientDeliveryState>();
     const now = new Date();
@@ -461,23 +629,113 @@ export class UnifiedDeliveryQueue extends EventEmitter {
     processingResult: Email,
     mode: EmailProcessingMode,
     route?: IEmailRoute,
+    enqueueOptions: IQueueEnqueueOptions = {},
   ): Promise<string> {
     this.assertMutable();
     if (!(processingResult instanceof Email)) {
       throw new Error('Queue processingResult must be an Email instance');
     }
 
+    if (enqueueOptions.requireDurable && !this.options.storageManager) {
+      throw new Error('Durable idempotent enqueue requires managed queue storage');
+    }
+    if (
+      (enqueueOptions.idempotencyKey === undefined)
+      !== (enqueueOptions.idempotencyFingerprint === undefined)
+    ) {
+      throw new Error('Queue idempotencyKey and idempotencyFingerprint must be provided together');
+    }
+    if (enqueueOptions.idempotencyKey !== undefined) {
+      if (this.options.storageManager && !this.options.storageManager.listPage) {
+        throw new Error(
+          'Managed idempotent enqueue requires paginated managed queue storage',
+        );
+      }
+      if (!enqueueOptions.idempotencyKey || enqueueOptions.idempotencyKey.length > 512) {
+        throw new Error('Queue idempotencyKey must contain 1 to 512 characters');
+      }
+      if (!/^[a-f0-9]{64}$/.test(enqueueOptions.idempotencyFingerprint!)) {
+        throw new Error('Queue idempotencyFingerprint must be a lowercase SHA-256 digest');
+      }
+
+      const id = this.idempotentQueueId(enqueueOptions.idempotencyKey);
+      const existing = this.queue.get(id);
+      if (existing) {
+        if (
+          existing.idempotencyKey !== enqueueOptions.idempotencyKey
+          || existing.idempotencyFingerprint !== enqueueOptions.idempotencyFingerprint
+        ) {
+          throw new Error(`Queue idempotency key collision for ${id}`);
+        }
+        return id;
+      }
+
+      const activeFlight = this.idempotentEnqueueFlights.get(id);
+      if (activeFlight) {
+        if (activeFlight.fingerprint !== enqueueOptions.idempotencyFingerprint) {
+          throw new Error(`Queue idempotency key collision for ${id}`);
+        }
+        return activeFlight.promise;
+      }
+      const flight = (async () => {
+        const tombstone = await this.readIdempotencyTombstone(id);
+        if (tombstone) {
+          if (
+            tombstone.idempotencyKey !== enqueueOptions.idempotencyKey
+            || tombstone.idempotencyFingerprint !== enqueueOptions.idempotencyFingerprint
+          ) {
+            throw new Error(`Queue idempotency key collision for ${id}`);
+          }
+          return id;
+        }
+        return this.enqueueNew(processingResult, mode, route, {
+          id,
+          idempotencyKey: enqueueOptions.idempotencyKey!,
+          idempotencyFingerprint: enqueueOptions.idempotencyFingerprint!,
+        });
+      })();
+      this.idempotentEnqueueFlights.set(id, {
+        fingerprint: enqueueOptions.idempotencyFingerprint!,
+        promise: flight,
+      });
+      try {
+        return await flight;
+      } finally {
+        if (this.idempotentEnqueueFlights.get(id)?.promise === flight) {
+          this.idempotentEnqueueFlights.delete(id);
+        }
+      }
+    }
+
+    return this.enqueueNew(processingResult, mode, route);
+  }
+
+  private async enqueueNew(
+    processingResult: Email,
+    mode: EmailProcessingMode,
+    route?: IEmailRoute,
+    idempotency?: {
+      id: string;
+      idempotencyKey: string;
+      idempotencyFingerprint: string;
+    },
+  ): Promise<string> {
     // Check if queue is full
     if (this.queue.size >= this.options.maxQueueSize) {
       throw new Error('Queue is full');
     }
     
     // Generate a unique ID
-    const id = `${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
+    const id = idempotency?.id
+      || `${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
     
     // Create queue item
     const item: IQueueItem = {
       id,
+      ...(idempotency ? {
+        idempotencyKey: idempotency.idempotencyKey,
+        idempotencyFingerprint: idempotency.idempotencyFingerprint,
+      } : {}),
       processingMode: mode,
       processingResult,
       attachmentCount: processingResult.attachments.length,
@@ -496,6 +754,13 @@ export class UnifiedDeliveryQueue extends EventEmitter {
     
     // Commit durable state before exposing the item in memory or emitting acceptance.
     await this.persistItem(item);
+    if (idempotency) {
+      await this.persistIdempotencyTombstone(
+        id,
+        idempotency.idempotencyKey,
+        idempotency.idempotencyFingerprint,
+      );
+    }
     this.queue.set(id, item);
     
     // Update statistics
@@ -978,6 +1243,13 @@ export class UnifiedDeliveryQueue extends EventEmitter {
       return false;
     }
     
+    if (item.idempotencyKey && item.idempotencyFingerprint) {
+      await this.persistIdempotencyTombstone(
+        item.id,
+        item.idempotencyKey,
+        item.idempotencyFingerprint,
+      );
+    }
     // Delete durable state before making the removal visible in memory.
     await this.removeItemFromStorage(id);
     this.queue.delete(id);
@@ -1008,6 +1280,10 @@ export class UnifiedDeliveryQueue extends EventEmitter {
     return {
       schemaVersion: 2,
       id: item.id,
+      ...(item.idempotencyKey !== undefined ? {
+        idempotencyKey: item.idempotencyKey,
+        idempotencyFingerprint: item.idempotencyFingerprint,
+      } : {}),
       processingMode: item.processingMode,
       processingResult: {
         options: item.processingResult.toEmailOptions(),
@@ -1081,6 +1357,27 @@ export class UnifiedDeliveryQueue extends EventEmitter {
     if (!parsed || typeof parsed !== 'object' || typeof parsed.id !== 'string' || !parsed.id) {
       throw new Error(`Invalid managed queue item at ${key}: missing id`);
     }
+    if (
+      parsed.idempotencyKey !== undefined
+      && (
+        typeof parsed.idempotencyKey !== 'string'
+        || !parsed.idempotencyKey
+        || parsed.idempotencyKey.length > 512
+        || typeof parsed.idempotencyFingerprint !== 'string'
+        || !/^[a-f0-9]{64}$/.test(parsed.idempotencyFingerprint)
+      )
+    ) {
+      throw new Error(`Invalid managed queue item at ${key}: invalid idempotency metadata`);
+    }
+    if (parsed.idempotencyKey === undefined && parsed.idempotencyFingerprint !== undefined) {
+      throw new Error(`Invalid managed queue item at ${key}: orphaned idempotency fingerprint`);
+    }
+    if (parsed.idempotencyKey !== undefined) {
+      const expectedId = this.idempotentQueueId(parsed.idempotencyKey);
+      if (parsed.id !== expectedId) {
+        throw new Error(`Invalid managed queue item at ${key}: idempotency id mismatch`);
+      }
+    }
     if (parsed.schemaVersion !== 1 && parsed.schemaVersion !== 2) {
       throw new Error(`Invalid managed queue item at ${key}: unsupported schema version`);
     }
@@ -1362,6 +1659,10 @@ export class UnifiedDeliveryQueue extends EventEmitter {
 
     return {
       id: parsed.id,
+      ...(parsed.idempotencyKey !== undefined ? {
+        idempotencyKey: parsed.idempotencyKey,
+        idempotencyFingerprint: parsed.idempotencyFingerprint,
+      } : {}),
       processingMode: parsed.processingMode,
       processingResult,
       attachmentCount: parsed.attachmentCount ?? processingResult.attachments.length,
@@ -1551,6 +1852,7 @@ export class UnifiedDeliveryQueue extends EventEmitter {
     for (const id of idsToRemove) {
       await this.removeItem(id);
     }
+    await this.cleanupExpiredIdempotencyTombstones();
 
     if (idsToRemove.length > 0) {
       logger.log('info', `Cleaned up ${idsToRemove.length} old items from delivery queue`);

package/ts/mail/interfaces.storage.ts

@@ -9,6 +9,23 @@ export interface IStorageManager {
   get(key: string): Promise<string | null>;
   set(key: string, value: string): Promise<void>;
   list(prefix: string): Promise<string[]>;
+  /**
+   * Return at most `limit` keys from one backend-bounded page.
+   *
+   * Continuation tokens are opaque, scoped to the supplied prefix, and must
+   * only be passed back to the same storage implementation. A returned token
+   * must advance a non-overlapping traversal, remain valid after any keys in
+   * its page are deleted, and eventually exhaust an otherwise stable prefix.
+   * Offset-based tokens do not satisfy this deletion-stability requirement.
+   *
+   * This optional method preserves compatibility for existing callers;
+   * features which need bounded cleanup must require it explicitly and must
+   * never fall back to list().
+   */
+  listPage?(
+    prefix: string,
+    options: IStorageListPageOptions,
+  ): Promise<IStorageListPageResult>;
   delete(key: string): Promise<void>;
 }
 
@@ -17,18 +34,131 @@ export interface IBlobStorageManager {
   get(key: string): Promise<Buffer | null>;
   set(key: string, value: Buffer): Promise<void>;
   list(prefix: string): Promise<string[]>;
+  /** Backend-bounded counterpart to list(); see IStorageManager.listPage. */
+  listPage?(
+    prefix: string,
+    options: IStorageListPageOptions,
+  ): Promise<IStorageListPageResult>;
   delete(key: string): Promise<void>;
 }
 
+export interface IStorageListPageOptions {
+  /** Positive maximum number of keys the backend may return. */
+  readonly limit: number;
+  /** Opaque token returned by the immediately preceding page. */
+  readonly continuationToken?: string;
+}
+
+export interface IStorageListPageResult {
+  /** Unique keys under the requested prefix; length must not exceed limit. */
+  readonly keys: string[];
+  /** Omitted when the prefix has been exhausted. */
+  readonly continuationToken?: string;
+}
+
+/** Hard upper bound for one SmartMTA storage listing page. */
+export const SMARTMTA_STORAGE_LIST_PAGE_MAX_LIMIT = 1_000;
+
+function assertValidListPageOptions(options: IStorageListPageOptions): void {
+  if (
+    !Number.isInteger(options.limit)
+    || options.limit < 1
+    || options.limit > SMARTMTA_STORAGE_LIST_PAGE_MAX_LIMIT
+  ) {
+    throw new Error(
+      `Storage list page limit must be an integer from 1 to ${SMARTMTA_STORAGE_LIST_PAGE_MAX_LIMIT}`,
+    );
+  }
+  if (
+    options.continuationToken !== undefined
+    && (
+      typeof options.continuationToken !== 'string'
+      || options.continuationToken.length === 0
+    )
+  ) {
+    throw new Error('Storage list page continuation token must be a non-empty string');
+  }
+}
+
+function lowerBound(sortedKeys: readonly string[], target: string): number {
+  let low = 0;
+  let high = sortedKeys.length;
+  while (low < high) {
+    const middle = low + Math.floor((high - low) / 2);
+    if (sortedKeys[middle] < target) low = middle + 1;
+    else high = middle;
+  }
+  return low;
+}
+
+function upperBound(sortedKeys: readonly string[], target: string): number {
+  let low = 0;
+  let high = sortedKeys.length;
+  while (low < high) {
+    const middle = low + Math.floor((high - low) / 2);
+    if (sortedKeys[middle] <= target) low = middle + 1;
+    else high = middle;
+  }
+  return low;
+}
+
+function insertSortedKey(sortedKeys: string[], key: string): void {
+  sortedKeys.splice(lowerBound(sortedKeys, key), 0, key);
+}
+
+function removeSortedKey(sortedKeys: string[], key: string): void {
+  const index = lowerBound(sortedKeys, key);
+  if (sortedKeys[index] === key) sortedKeys.splice(index, 1);
+}
+
+function listSortedPrefixPage(
+  sortedKeys: readonly string[],
+  prefix: string,
+  options: IStorageListPageOptions,
+): IStorageListPageResult {
+  assertValidListPageOptions(options);
+  if (
+    options.continuationToken !== undefined
+    && !options.continuationToken.startsWith(prefix)
+  ) {
+    throw new Error('Storage list page continuation token does not belong to the prefix');
+  }
+  let index = options.continuationToken === undefined
+    ? lowerBound(sortedKeys, prefix)
+    : Math.max(
+        lowerBound(sortedKeys, prefix),
+        upperBound(sortedKeys, options.continuationToken),
+      );
+  const keys: string[] = [];
+  while (index < sortedKeys.length && keys.length < options.limit) {
+    const key = sortedKeys[index];
+    if (!key.startsWith(prefix)) break;
+    keys.push(key);
+    index++;
+  }
+  const hasMore = (
+    index < sortedKeys.length
+    && sortedKeys[index].startsWith(prefix)
+  );
+  return {
+    keys,
+    ...(hasMore && keys.length > 0
+      ? { continuationToken: keys[keys.length - 1] }
+      : {}),
+  };
+}
+
 /** In-memory small-state store for tests and explicitly non-durable runtimes. */
 export class MemoryStorageManager implements IStorageManager {
   private readonly entries = new Map<string, string>();
+  private readonly sortedKeys: string[] = [];
 
   public async get(key: string): Promise<string | null> {
     return this.entries.get(key) ?? null;
   }
 
   public async set(key: string, value: string): Promise<void> {
+    if (!this.entries.has(key)) insertSortedKey(this.sortedKeys, key);
     this.entries.set(key, value);
   }
 
@@ -36,14 +166,22 @@ export class MemoryStorageManager implements IStorageManager {
     return [...this.entries.keys()].filter((key) => key.startsWith(prefix)).sort();
   }
 
+  public async listPage(
+    prefix: string,
+    options: IStorageListPageOptions,
+  ): Promise<IStorageListPageResult> {
+    return listSortedPrefixPage(this.sortedKeys, prefix, options);
+  }
+
   public async delete(key: string): Promise<void> {
-    this.entries.delete(key);
+    if (this.entries.delete(key)) removeSortedKey(this.sortedKeys, key);
   }
 }
 
 /** In-memory blob store for tests and explicitly non-durable runtimes. */
 export class MemoryBlobStorageManager implements IBlobStorageManager {
   private readonly entries = new Map<string, Buffer>();
+  private readonly sortedKeys: string[] = [];
 
   public async get(key: string): Promise<Buffer | null> {
     const value = this.entries.get(key);
@@ -51,6 +189,7 @@ export class MemoryBlobStorageManager implements IBlobStorageManager {
   }
 
   public async set(key: string, value: Buffer): Promise<void> {
+    if (!this.entries.has(key)) insertSortedKey(this.sortedKeys, key);
     this.entries.set(key, Buffer.from(value));
   }
 
@@ -58,8 +197,15 @@ export class MemoryBlobStorageManager implements IBlobStorageManager {
     return [...this.entries.keys()].filter((key) => key.startsWith(prefix)).sort();
   }
 
+  public async listPage(
+    prefix: string,
+    options: IStorageListPageOptions,
+  ): Promise<IStorageListPageResult> {
+    return listSortedPrefixPage(this.sortedKeys, prefix, options);
+  }
+
   public async delete(key: string): Promise<void> {
-    this.entries.delete(key);
+    if (this.entries.delete(key)) removeSortedKey(this.sortedKeys, key);
   }
 }