@push.rocks/smartmta 9.4.0 → 9.5.0

package/changelog.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## 2026-07-31 - 9.5.0
+
+### Features
+
+- add a durable raw outbound MTA queue (raw-outbound)
+  - Add managed-only `UnifiedEmailServer.enqueueRawEmail()` with canonical CRLF-terminated exact RFC822 bytes, a separate canonical ordered SMTP envelope, stable idempotency, a 32 MiB default message limit, and a weighted global raw-operation memory budget.
+  - Store raw items as atomic schema V3 queue blobs while retaining only sanitized metadata in memory; preserve structured schema V2 writes and V1/V2 hydration unchanged.
+  - Serialize raw status mutations and removal per item, verify payload length/digest/envelope on every lazy reread, refuse processing-item removal, and fail closed on missing or corrupt payloads.
+  - Isolate all public queue/status snapshots, keep queue-only operations behind internal symbols, and route internal readiness by item ID so observers cannot mutate delivery inputs.
+  - Reuse MX, self-MX, recipient checkpoint, retry, bounce, RemoteIngress proxy/source-evidence, timeout, TLS, pooling, capture, and cleanup behavior for raw delivery.
+  - Extend Rust raw SMTP delivery with mandatory caller-supplied DKIM material, canonical-byte enforcement, and actual-sent-byte capture without UTF-8 conversion; keep unsigned queue bytes immutable across attempts and capture memory reserved through verification.
+  - Snapshot raw caller buffers synchronously, reserve queue capacity before persistence, share the weighted memory budget with direct raw sends, cap fixed raw-operation overhead, use bounded startup hydration, close outbound admission before shutdown, drain admitted enqueue/queue-scan/delivery/authentication work before storage and Rust teardown, and recover interrupted V3 records without dropping their payload.
+  - Add `smartMtaCapabilities.durableRawOutboundQueue` and document the schema V3 downgrade gate and deployment-wide single-writer prerequisite.
+
+## 2026-07-27 - 9.4.1
+
+### Fixes
+
+- dispatch newly queued delivery items immediately (delivery)
+  - Wake the normal queue eligibility scan after durable enqueue, resume, delivery completion, and rate-limit reset while preserving pause and retry timing.
+  - Serialize queue scan wakeups and bound shutdown when an active itemsReady observer does not settle.
+  - Use a fixed one-minute rate-limit window to avoid false throttling on freshly started delivery processing.
+
 ## 2026-07-27 - 9.4.0
 
 ### Features

package/dist_ts/00_commitinfo_data.js

@@ -3,7 +3,7 @@
  */
 export const commitinfo = {
     name: '@push.rocks/smartmta',
-    version: '9.4.0',
+    version: '9.5.0',
     description: 'A high-performance, enterprise-grade Mail Transfer Agent (MTA) built from scratch in TypeScript with Rust acceleration.'
 };
 //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiMDBfY29tbWl0aW5mb19kYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vdHMvMDBfY29tbWl0aW5mb19kYXRhLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBOztHQUVHO0FBQ0gsTUFBTSxDQUFDLE1BQU0sVUFBVSxHQUFHO0lBQ3hCLElBQUksRUFBRSxzQkFBc0I7SUFDNUIsT0FBTyxFQUFFLE9BQU87SUFDaEIsV0FBVyxFQUFFLHlIQUF5SDtDQUN2SSxDQUFBIn0=

package/dist_ts/index.d.ts

@@ -15,4 +15,5 @@ export declare const smartMtaCapabilities: Readonly<{
     readonly exactAcceptedEnvelopeDispatch: true;
     readonly terminalLocalDeliverActions: true;
     readonly smtpAuthAccountsAndScopes: true;
+    readonly durableRawOutboundQueue: true;
 }>;

package/dist_ts/index.js

@@ -15,5 +15,6 @@ export const smartMtaCapabilities = Object.freeze({
     exactAcceptedEnvelopeDispatch: true,
     terminalLocalDeliverActions: true,
     smtpAuthAccountsAndScopes: true,
+    durableRawOutboundQueue: true,
 });
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxjQUFjLHlCQUF5QixDQUFDO0FBQ3hDLGNBQWMsaUJBQWlCLENBQUM7QUFDaEMsY0FBYyxxQkFBcUIsQ0FBQztBQUNwQyxPQUFPLEVBQUUsVUFBVSxFQUFFLE1BQU0sRUFBRSxNQUFNLGFBQWEsQ0FBQztBQUVqRCxNQUFNLENBQUMsTUFBTSxvQkFBb0IsR0FBRyxNQUFNLENBQUMsTUFBTSxDQUFDO0lBQ2hELDBCQUEwQixFQUFFLElBQUk7SUFDaEMscUJBQXFCLEVBQUUsSUFBSTtJQUMzQiwwQkFBMEIsRUFBRSxJQUFJO0lBQ2hDLHFCQUFxQixFQUFFLElBQUk7SUFDM0Isb0JBQW9CLEVBQUUsSUFBSTtJQUMxQixpQkFBaUIsRUFBRSxJQUFJO0lBQ3ZCLG1CQUFtQixFQUFFLElBQUk7SUFDekIsbUJBQW1CLEVBQUUsSUFBSTtJQUN6Qiw0QkFBNEIsRUFBRSxJQUFJO0lBQ2xDLDZCQUE2QixFQUFFLElBQUk7SUFDbkMsMkJBQTJCLEVBQUUsSUFBSTtJQUNqQyx5QkFBeUIsRUFBRSxJQUFJO0NBQ3ZCLENBQUMsQ0FBQyJ9
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxjQUFjLHlCQUF5QixDQUFDO0FBQ3hDLGNBQWMsaUJBQWlCLENBQUM7QUFDaEMsY0FBYyxxQkFBcUIsQ0FBQztBQUNwQyxPQUFPLEVBQUUsVUFBVSxFQUFFLE1BQU0sRUFBRSxNQUFNLGFBQWEsQ0FBQztBQUVqRCxNQUFNLENBQUMsTUFBTSxvQkFBb0IsR0FBRyxNQUFNLENBQUMsTUFBTSxDQUFDO0lBQ2hELDBCQUEwQixFQUFFLElBQUk7SUFDaEMscUJBQXFCLEVBQUUsSUFBSTtJQUMzQiwwQkFBMEIsRUFBRSxJQUFJO0lBQ2hDLHFCQUFxQixFQUFFLElBQUk7SUFDM0Isb0JBQW9CLEVBQUUsSUFBSTtJQUMxQixpQkFBaUIsRUFBRSxJQUFJO0lBQ3ZCLG1CQUFtQixFQUFFLElBQUk7SUFDekIsbUJBQW1CLEVBQUUsSUFBSTtJQUN6Qiw0QkFBNEIsRUFBRSxJQUFJO0lBQ2xDLDZCQUE2QixFQUFFLElBQUk7SUFDbkMsMkJBQTJCLEVBQUUsSUFBSTtJQUNqQyx5QkFBeUIsRUFBRSxJQUFJO0lBQy9CLHVCQUF1QixFQUFFLElBQUk7Q0FDckIsQ0FBQyxDQUFDIn0=

package/dist_ts/mail/delivery/classes.delivery.queue.d.ts

@@ -1,9 +1,11 @@
 import { EventEmitter } from 'node:events';
-import type { IBlobStorageManager } from '../interfaces.storage.js';
+import { type IBlobStorageManager } from '../interfaces.storage.js';
 import { type EmailProcessingMode } from './interfaces.js';
 import type { ISmtpTransactionAttempt } from './interfaces.js';
 import type { IEmailRoute } from '../routing/interfaces.js';
 import { Email } from '../core/classes.email.js';
+import { type IRawSmtpEnvelope } from './functions.raw-email.js';
+import { kQueueCloseAdmission, kQueueCreateItemSnapshot, kQueueEnqueueRaw, kQueueReserveRawAccess, kQueueSetReadyItemsHandler, kQueueWithRawMessage } from './symbols.delivery.queue.js';
 /**
  * Queue item status
  */
@@ -47,6 +49,14 @@ export interface IQueueItem {
     idempotencyFingerprint?: string;
     processingMode: EmailProcessingMode;
     processingResult: Email;
+    /** Present only for durable exact-byte outbound queue items. */
+    messageFormat?: 'raw-rfc822';
+    /** SHA-256 of the immutable unsigned RFC822 bytes. */
+    rawMessageSha256?: string;
+    /** Exact immutable unsigned RFC822 byte length. */
+    rawMessageLength?: number;
+    /** Canonical SMTP envelope, independent from message headers. */
+    smtpEnvelope?: IRawSmtpEnvelope;
     /** Materialized attachment count for queue/detail consumers. */
     attachmentCount?: number;
     route?: IEmailRoute;
@@ -64,6 +74,8 @@ export interface IQueueItem {
     smtpTransactions?: ISmtpTransactionAttempt[];
 }
 export declare const ACCEPTED_ENVELOPE_IDEMPOTENCY_RETENTION_MS: number;
+export declare const DEFAULT_MAX_RAW_MESSAGE_BYTES: number;
+export declare const DEFAULT_RAW_IN_FLIGHT_BYTE_BUDGET: number;
 export interface IQueueFailureOptions {
     /** Unknown failures default to retryable so temporary internal faults do not lose mail. */
     retryable?: boolean;
@@ -91,6 +103,10 @@ export interface IQueueOptions {
     checkInterval?: number;
     maxQueueSize?: number;
     maxPerDestination?: number;
+    /** Maximum exact RFC822 bytes accepted by enqueueRawEmail. Defaults to 32 MiB. */
+    maxRawMessageBytes?: number;
+    /** Conservative memory budget for active raw operations. Defaults to 192 MiB. */
+    rawInFlightByteBudget?: number;
     maxRetries?: number;
     baseRetryDelay?: number;
     maxRetryDelay?: number;
@@ -103,6 +119,11 @@ export interface IQueueEnqueueOptions {
     /** Fail closed unless the queue is backed by managed blob storage. */
     requireDurable?: boolean;
 }
+interface IRawQueueEnqueueOptions {
+    rawMessage: Buffer;
+    envelope: IRawSmtpEnvelope;
+    idempotencyKey: string;
+}
 export interface IQueueStats {
     queueSize: number;
     status: {
@@ -131,23 +152,46 @@ export declare class UnifiedDeliveryQueue extends EventEmitter {
     private queue;
     private checkTimer?;
     private cleanupTimer?;
+    private cleanupFlight?;
     private stats;
     private processing;
+    private processingScanPromise?;
+    private processingRescanRequested;
     private totalProcessed;
     private ownsManagedPrefix;
     private lifecycleState;
     private readonly idempotentEnqueueFlights;
     private idempotencyTombstoneCleanupCursor?;
+    private readonly itemMutationFlights;
+    private rawBytesInFlight;
+    private rawByteWaiters;
+    private rawReservationCount;
+    private rawReservationDrainWaiters;
+    private readyItemsHandler?;
+    private acceptingNewItems;
+    private acceptingMutations;
+    private acceptingRawReservations;
+    private readonly enqueueFlights;
+    private pendingNewItemCount;
+    private shutdownFlight?;
     /**
      * Create a new unified delivery queue
      * @param options Queue options
      */
     constructor(options: IQueueOptions);
     private assertMutable;
+    private assertAcceptingNewItems;
+    private trackEnqueueFlight;
     /** Whether queue entries are committed through caller-managed blob storage. */
     hasManagedStorage(): boolean;
     /** Whether managed storage can list a backend-bounded page of keys. */
     hasPaginatedManagedStorage(): boolean;
+    [kQueueCloseAdmission](): void;
+    [kQueueSetReadyItemsHandler](handlerArg?: (itemIds: string[]) => Promise<void>): void;
+    private cloneEmail;
+    [kQueueCreateItemSnapshot](itemArg: IQueueItem): IQueueItem;
+    private emitItemEventSafely;
+    private emitReadyItemsSafely;
     private idempotentQueueId;
     private idempotencyTombstonePrefix;
     private idempotencyTombstoneKey;
@@ -169,6 +213,7 @@ export declare class UnifiedDeliveryQueue extends EventEmitter {
      * Start queue processing
      */
     private startProcessing;
+    private runPeriodicCleanup;
     /**
      * Stop queue processing
      */
@@ -177,6 +222,12 @@ export declare class UnifiedDeliveryQueue extends EventEmitter {
      * Check for items that need to be processed
      */
     private processQueue;
+    /**
+     * Wake the normal eligibility scan without bypassing pause or retry timing.
+     * The interval remains the recovery path for retries and any missed wakeup.
+     */
+    wake(): void;
+    private runProcessingScans;
     /**
      * Add an item to the queue
      * @param processingResult Processing result to queue
@@ -184,6 +235,10 @@ export declare class UnifiedDeliveryQueue extends EventEmitter {
      * @param route Email route
      */
     enqueue(processingResult: Email, mode: EmailProcessingMode, route?: IEmailRoute, enqueueOptions?: IQueueEnqueueOptions): Promise<string>;
+    private enqueueAccepted;
+    [kQueueEnqueueRaw](optionsArg: IRawQueueEnqueueOptions): Promise<string>;
+    private enqueueRawAccepted;
+    private enqueueIdempotent;
     private enqueueNew;
     /**
      * Get an item from the queue
@@ -244,8 +299,21 @@ export declare class UnifiedDeliveryQueue extends EventEmitter {
     removeItem(id: string): Promise<boolean>;
     private storageKey;
     private persistItem;
+    private persistCurrentItem;
     private serializeItem;
+    private withItemMutex;
     private applyPersistedMutation;
+    private createRawIntegrityError;
+    private readVerifiedRawMessage;
+    private claimRawReservation;
+    private createRawByteRelease;
+    private reserveRawBytes;
+    private reserveRawBytesImmediately;
+    [kQueueReserveRawAccess](rawMessageLengthArg: number): () => void;
+    private releaseRawBytes;
+    private withRawByteReservation;
+    private waitForRawReservationsToDrain;
+    [kQueueWithRawMessage]<T>(itemIdArg: string, callbackArg: (rawMessage: Buffer) => Promise<T>): Promise<T>;
     private removeItemFromStorage;
     private deserializeItem;
     private loadFromStorage;
@@ -274,4 +342,6 @@ export declare class UnifiedDeliveryQueue extends EventEmitter {
      * Shutdown the queue
      */
     shutdown(): Promise<void>;
+    private shutdownOnce;
 }
+export {};