@spine-event-engine/server 2.0.0-snapshot.23 → 2.0.0-snapshot.25

package/dist/delivery/delivery.js

@@ -11,6 +11,7 @@
  * or implied. See the License for the specific language governing permissions and limitations under
  * the License.
  */
+var _a;
 import { Time } from "@spine-event-engine/core/time";
 import { clone, create } from "@bufbuild/protobuf";
 import { AnySchema, TimestampSchema } from "@bufbuild/protobuf/wkt";
@@ -22,10 +23,16 @@ import { Inbox, InboxTargets } from "./inbox.js";
 import { InboxStorage, InboxStorageClock } from "./inbox-storage.js";
 import { ShardIndex } from "./shard-index.js";
 import { ShardedWorkRegistry } from "./sharded-work-registry.js";
+import { deliveryTenantKey } from "./delivery-tenant.js";
 import { withDeliveryCommitFence } from "../repository/commit-fence.js";
+import { receptionObservation } from "./reception-observation.js";
+import { AgentDeliveryScope } from "./agent-delivery-scope.js";
+import { emitServerError } from "../server/server-log.js";
 const deliveryFailureLimit = 100;
+const controlledMessageSettlers = new WeakMap();
+const dispatchNotAdmitted = Symbol("Delivery recipient closed before dispatch");
 /**
- * Executes direct Inbox delivery for one complete worker identity.
+ * Runs one retained direct-delivery worker and its durable Inbox.
  */
 export class Delivery {
     // prettier-ignore
@@ -62,18 +69,146 @@ export class Delivery {
      */
     shards;
     #monitor;
+    #directPending = new Map();
+    #logger;
+    #backgroundObserved = new WeakSet();
+    #callerObserved = new WeakSet();
+    #activeDirect = new Map();
+    #activeRuns = new Set();
+    #directDrains = new Map();
+    #pendingDirectWake = new Set();
+    #replay;
+    #acceptReplay;
+    #closing = false;
+    #startupDraining = false;
+    #closeAttempt;
+    #closingRuns;
+    /**
+     * Sets the complete direct fallback dispatcher before receiving work.
+     *
+     * @internal
+     * @param onReplay Callback that replays a ready Inbox route.
+     * @param onAccept Reports whether a live recipient currently serves the row.
+     */
+    setReplay(onReplay, onAccept) {
+        this.#replay = onReplay;
+        this.#acceptReplay = onAccept;
+    }
+    /**
+     * Waits for every retained direct shard run to release its session.
+     *
+     * @internal
+     * @returns Completion after retained shard sessions release.
+     */
+    async whenIdle() {
+        const failures = [];
+        while (this.#activeRuns.size > 0) {
+            const active = [...this.#activeRuns];
+            const settled = await Promise.allSettled(active.map(({ completed }) => completed));
+            settled.forEach((outcome, index) => {
+                const failure = active[index]?.releaseFailure;
+                if (failure !== undefined)
+                    failures.push(failure);
+                if (outcome.status === "rejected")
+                    failures.push(outcome.reason);
+            });
+        }
+        if (failures.length === 1)
+            throw failures[0];
+        if (failures.length > 1)
+            throw new AggregateError(failures, "Delivery idle wait failed.");
+    }
+    /**
+     * Stops direct admission and drains all entered shard work.
+     *
+     * @internal
+     * @returns Completion after admitted work and retained cleanup settle.
+     */
+    close() {
+        if (this.#closeAttempt !== undefined)
+            return this.#closeAttempt;
+        this.beginClose();
+        const active = this.#closingRuns ?? [];
+        this.#closeAttempt = Promise.allSettled(active.map(({ completed }) => completed)).then(async (settled) => {
+            const failures = [];
+            settled.forEach((result, index) => {
+                const releaseFailure = active[index]?.releaseFailure;
+                if (releaseFailure !== undefined)
+                    failures.push(releaseFailure);
+                if (result.status === "rejected")
+                    failures.push(result.reason);
+            });
+            const storage = await Promise.allSettled([
+                this.inbox instanceof Inbox ? this.inbox.storage.close() : Promise.resolve(),
+                this.shards instanceof ShardedWorkRegistry ? this.shards.close() : Promise.resolve(),
+            ]);
+            for (const result of storage)
+                if (result.status === "rejected")
+                    failures.push(result.reason);
+            if (failures.length === 1)
+                throw failures[0];
+            if (failures.length)
+                throw new AggregateError(failures, "Delivery close failed.");
+        });
+        return this.#closeAttempt;
+    }
+    /**
+     * Closes admission synchronously and retains admitted release outcomes.
+     *
+     * @internal
+     */
+    beginClose() {
+        if (this.#closing)
+            return;
+        this.#closing = true;
+        this.#closingRuns ??= [...this.#activeRuns];
+        for (const active of this.#closingRuns)
+            active.drain?.stop(new Error("Delivery closed."));
+    }
+    /**
+     * Marks public runs as unavailable while preserving exact admitted local handoffs.
+     *
+     * @internal
+     */
+    beginStartupDrain() {
+        if (this.#startupDraining || this.#closing)
+            return;
+        this.#startupDraining = true;
+        this.#closingRuns = [...this.#activeRuns];
+    }
+    /**
+     * Notifies an admitted shard run to scan newly eligible durable rows.
+     *
+     * @internal
+     * @param shard Delivery shard identifier.
+     */
+    wake(shard) {
+        if (this.#closing || (this.#startupDraining && !this.#hasPending(shard.key())))
+            return;
+        const drain = this.#directDrains.get(shard.key());
+        if (drain !== undefined) {
+            if (!drain.wake())
+                this.#pendingDirectWake.add(shard.key());
+            return;
+        }
+        if (this.#replay !== undefined)
+            this.#observeBackground(this.#startDirect(shard), shard);
+    }
     /**
      * Creates direct delivery from immutable options.
      *
      * @param options The delivery configuration.
      */
     constructor(options) {
+        if (options.context.multitenant)
+            throw new TypeError("Delivery requires a single-tenant storage context.");
         if (options.worker !== undefined &&
             options.node !== undefined &&
             options.worker.nodeId?.value !== options.node) {
             throw new Error("Delivery worker node must match the configured delivery node.");
         }
-        this.context = Delivery.#contextSnapshot(options.context);
+        this.context = _a.#contextSnapshot(options.context);
+        this.#logger = options.logger;
         this.storageFactory = options.storageFactory;
         this.strategy = options.strategy ?? { shardCount: 1, shardFor: () => ShardIndex.single() };
         this.worker =
@@ -118,19 +253,42 @@ export class Delivery {
      */
     runControlled(options) {
         const shard = options.shard;
-        return this.drain(shard, {
-            onMessage: options.onMessage,
-            ...(options.acceptMessage === undefined ? {} : { acceptMessage: options.acceptMessage }),
-            operation: { signal: options.signal },
-        }).then((run) => Object.freeze({
-            status: run.status === "SKIPPED"
-                ? "SKIPPED"
-                : run.status === "FAILED"
-                    ? "FAILED"
-                    : run.status === "STOPPED"
-                        ? "STOPPED"
-                        : "COMPLETED",
-        }));
+        const drainOptions = this.#controlledOptions(options);
+        if (options.onMessageSettled !== undefined)
+            controlledMessageSettlers.set(drainOptions, options.onMessageSettled);
+        const completed = this.#replay === undefined
+            ? this.drain(shard, drainOptions)
+            : this.#startShared(shard, drainOptions);
+        return completed
+            .then((run) => {
+            options.onDrainResult?.(run);
+            return Object.freeze({
+                status: run.status === "SKIPPED"
+                    ? "SKIPPED"
+                    : run.status === "FAILED"
+                        ? "FAILED"
+                        : run.status === "STOPPED"
+                            ? "STOPPED"
+                            : "COMPLETED",
+            });
+        })
+            .finally(() => {
+            controlledMessageSettlers.delete(drainOptions);
+        });
+    }
+    /**
+     * Builds the exact controlled options for an endpoint or retained replay run.
+     * @param options Controlled run input.
+     * @returns The options passed to one physical shard drain.
+     */
+    #controlledOptions(options) {
+        return this.#replay === undefined
+            ? {
+                onMessage: options.onMessage,
+                ...(options.acceptMessage === undefined ? {} : { acceptMessage: options.acceptMessage }),
+                operation: { signal: options.signal },
+            }
+            : { ...this.#directOptions(), operation: { signal: options.signal } };
     }
     /**
      * Delivers the selected message's shard for local handoff integration.
@@ -155,22 +313,148 @@ export class Delivery {
             : { name: context.name, multitenant: false });
     }
     async #drainDirect(message, input) {
-        let acknowledged = false;
-        const run = await this.drain(message.shard, {
-            onMessage: input.onMessage,
-            ...(input.acceptMessage === undefined ? {} : { acceptMessage: input.acceptMessage }),
-            onDelivered: (next) => {
-                acknowledged ||=
-                    next.id.value === message.id.value && next.id.shard.key() === message.id.shard.key();
-                input.onDelivered?.(next);
-            },
-            onDuplicateRemoved: (next) => {
-                acknowledged ||=
-                    next.id.value === message.id.value && next.id.shard.key() === message.id.shard.key();
-                input.onDuplicateRemoved?.(next);
+        const key = _a.#messageKey(message);
+        const pending = this.#directPending.get(key);
+        if (pending !== undefined)
+            return pending.completed;
+        const selected = Promise.withResolvers();
+        const settle = (outcome) => {
+            if (this.#directPending.get(key)?.settle !== settle)
+                return false;
+            this.#directPending.delete(key);
+            selected.resolve(Object.freeze({ acknowledged: outcome.kind === "acknowledged", outcome }));
+            return true;
+        };
+        this.#directPending.set(key, {
+            shardKey: message.shard.key(),
+            input,
+            settle,
+            completed: selected.promise,
+        });
+        const completed = this.#startDirect(message.shard);
+        this.#observeSelectedRun(message.shard, key, completed, settle);
+        return selected.promise;
+    }
+    /**
+     * Transfers an unresolved exact-row observer to a successor shard run.
+     */
+    #observeSelectedRun(shard, key, completed, settle) {
+        void completed.then((run) => {
+            if (!this.#directPending.has(key))
+                return;
+            const successor = this.#activeDirect.get(shard.key())?.completed;
+            if (successor !== undefined && successor !== completed) {
+                this.#observeSelectedRun(shard, key, successor, settle);
+            }
+            else
+                settle({ kind: "unresolved", run });
+        }, (error) => {
+            if (settle({ kind: "failed", error }))
+                this.#callerObserved.add(completed);
+        });
+    }
+    static #messageKey(message) {
+        return JSON.stringify([
+            deliveryTenantKey(message),
+            message.id.value,
+            message.id.shard.index,
+            message.id.shard.ofTotal,
+        ]);
+    }
+    /**
+     * Starts or wakes the retained direct run for one shard.
+     */
+    #startDirect(shard) {
+        if (this.#closing || (this.#startupDraining && !this.#hasPending(shard.key())))
+            return Promise.resolve(result("STOPPED"));
+        return this.#startShared(shard, this.#directOptions());
+    }
+    /**
+     * Checks whether an exact accepted handoff needs this shard.
+     * @param shardKey Canonical shard key.
+     * @returns Whether an exact pending observer remains.
+     */
+    #hasPending(shardKey) {
+        return [...this.#directPending.values()].some((pending) => pending.shardKey === shardKey);
+    }
+    /**
+     * Starts or joins the one retained shard run used by local posts and supervision.
+     * @param shard Shard selected by the accepted message or supervisor.
+     * @param options Complete drain callbacks for a newly admitted run.
+     * @returns The terminal run shared by both entry paths.
+     */
+    #startShared(shard, options) {
+        const key = shard.key();
+        if (this.#startupDraining && !this.#hasPending(key))
+            return Promise.resolve(result("STOPPED"));
+        const active = this.#activeDirect.get(key);
+        if (active !== undefined) {
+            active.wake();
+            return active.completed;
+        }
+        const completed = this.#trackDrain(shard, options, true);
+        this.#activeDirect.set(key, {
+            completed,
+            wake: () => {
+                const drain = this.#directDrains.get(key);
+                if (drain === undefined)
+                    this.#pendingDirectWake.add(key);
+                else if (!drain.wake())
+                    this.#pendingDirectWake.add(key);
             },
         });
-        return Object.freeze({ run, acknowledged });
+        void completed
+            .finally(() => {
+            this.#activeDirect.delete(key);
+            this.#directDrains.delete(key);
+            if (this.#pendingDirectWake.delete(key) && !this.#closing)
+                this.#observeBackground(this.#startDirect(shard), shard);
+        })
+            .catch(() => undefined);
+        return completed;
+    }
+    /**
+     * Reports a detached local shard failure without changing its result.
+     * @param completed Detached shard run.
+     * @param shard Failed shard.
+     */
+    #observeBackground(completed, shard) {
+        if (this.#backgroundObserved.has(completed))
+            return;
+        this.#backgroundObserved.add(completed);
+        void completed.catch(async (error) => {
+            // Give exact-row observers of this rejection a chance to settle first.
+            await Promise.resolve();
+            if (this.#closing || this.#logger === undefined || this.#callerObserved.has(completed))
+                return;
+            emitServerError(this.#logger, "Background Delivery failed.", {
+                shardId: shard.key(),
+                errorName: error instanceof Error ? error.name : "UnknownError",
+            });
+        });
+    }
+    /**
+     * Dispatches each selected direct row through its registered endpoint.
+     */
+    #directOptions() {
+        const request = (message) => this.#directPending.get(_a.#messageKey(message));
+        return {
+            onMessage: (message) => {
+                const selected = request(message);
+                return selected === undefined ? this.#replay?.(message) : selected.input.onMessage(message);
+            },
+            acceptMessage: (message) => {
+                const selected = request(message);
+                return selected === undefined
+                    ? !this.#startupDraining &&
+                        this.#replay !== undefined &&
+                        (this.#acceptReplay?.(message) ?? true)
+                    : (selected.input.acceptMessage?.(message) ?? true);
+            },
+            onDelivered: (message) => request(message)?.input.onDelivered?.(message),
+            onDuplicateRemoved: (message) => request(message)?.input.onDuplicateRemoved?.(message),
+            onMessageOutcome: (message, outcome) => request(message)?.settle(outcome),
+        };
     }
     /**
      * Executes one owned shard drain while containing monitor and endpoint failures.
@@ -179,8 +463,37 @@ export class Delivery {
      * @param options The endpoint and optional operation signal.
      * @returns The terminal direct-delivery result.
      */
-    async drain(shard, options) {
-        if (options.operation?.signal?.aborted)
+    drain(shard, options) {
+        return this.#trackDrain(shard, options, false);
+    }
+    /**
+     * Tracks one complete physical drain before opening its shard session.
+     * @param shard Selected shard.
+     * @param options Complete drain callbacks.
+     * @param shared Whether the run serves the shared replay route.
+     * @returns Completion after shard release.
+     */
+    #trackDrain(shard, options, shared) {
+        if (this.#closing || (this.#startupDraining && (!shared || !this.#hasPending(shard.key()))))
+            return Promise.resolve(result("STOPPED"));
+        const completion = Promise.withResolvers();
+        const active = { completed: completion.promise, shared };
+        this.#activeRuns.add(active);
+        if (this.#startupDraining)
+            this.#closingRuns?.push(active);
+        void this.#drainAdmitted(shard, options, active).then(completion.resolve, completion.reject);
+        void active.completed.finally(() => this.#activeRuns.delete(active)).catch(() => undefined);
+        return active.completed;
+    }
+    /**
+     * Completes shard acquisition, callbacks, and release for an admitted run.
+     * @param shard Selected shard.
+     * @param options Complete drain callbacks.
+     * @param active Admitted run record.
+     * @returns Terminal result after release.
+     */
+    async #drainAdmitted(shard, options, active) {
+        if (this.#shouldStop(options))
             return result("STOPPED");
         let session;
         try {
@@ -194,11 +507,41 @@ export class Delivery {
             await safely(async () => (await this.#monitor.onShardAlreadyPicked(new AlreadyPickedUp(shard))).execute());
             return result("SKIPPED");
         }
-        if (options.operation?.signal?.aborted) {
-            const released = await safelyValue(() => this.shards.release(session), false);
+        if (this.#shouldStop(options))
+            return this.#releaseStoppedSession(session, active);
+        return this.#runSession(shard, options, session, active);
+    }
+    /**
+     * Releases a session acquired after direct admission closed.
+     *
+     * @param session The acquired session to release.
+     * @param active The admitted run retaining release failure.
+     * @returns The stopped or failed direct-delivery result.
+     */
+    async #releaseStoppedSession(session, active) {
+        try {
+            const released = await this.shards.release(session);
+            if (!released)
+                active.releaseFailure = new Error("Shard release was not confirmed.");
             return result(released ? "STOPPED" : "FAILED");
         }
-        return new DeliveryDrain({
+        catch (error) {
+            active.releaseFailure =
+                error instanceof Error ? error : new Error("Shard release failed.", { cause: error });
+            return result("FAILED");
+        }
+    }
+    /**
+     * Reads current admission fences after asynchronous shard acquisition.
+     */
+    #shouldStop(options) {
+        return this.#closing || options.operation?.signal?.aborted === true;
+    }
+    /**
+     * Tracks an admitted direct session through its complete drain.
+     */
+    #runSession(shard, options, session, active) {
+        const drain = new DeliveryDrain({
             inbox: this.inbox,
             monitor: this.#monitor,
             options,
@@ -206,22 +549,58 @@ export class Delivery {
             session,
             shards: this.shards,
             shard,
-        }).run();
+        });
+        active.drain = drain;
+        if (active.shared) {
+            this.#directDrains.set(shard.key(), drain);
+            if (this.#pendingDirectWake.delete(shard.key()))
+                drain.wake();
+        }
+        const completed = drain.run().finally(() => {
+            active.releaseFailure = drain.releaseFailure;
+        });
+        void completed
+            .finally(() => {
+            if (active.shared && this.#directDrains.get(shard.key()) === drain)
+                this.#directDrains.delete(shard.key());
+        })
+            .catch(() => undefined);
+        return completed;
     }
 }
+_a = Delivery;
 /**
  * Processes inbox pages while retaining one shard session.
  */
 class DeliveryDrain {
     input;
     #statistics = counts();
+    #agentController = new AbortController();
+    #liveAgents = new Set();
     #failures = [];
     #deduplication;
     #blockedTargets = new Set();
+    #capacityBlockedTargets = new Set();
     #removedInPage = new Set();
+    #messageOutcomes = new WeakMap();
+    #wakeGeneration = 0;
+    #wakeGate = Promise.withResolvers();
+    #sessionSerial = Promise.resolve();
+    #renewalTimer;
     #current;
     #ownershipLost = false;
     #failureLimitReached = false;
+    #stopped = false;
+    #reading = true;
+    #releaseFailure;
+    /**
+     * Returns a failed session release to an active direct close.
+     *
+     * @returns The original shard-release failure, if one occurred.
+     */
+    get releaseFailure() {
+        return this.#releaseFailure;
+    }
     /**
      * Creates a drain for one acquired shard session.
      *
@@ -232,6 +611,84 @@ class DeliveryDrain {
         this.#current = input.session;
         this.#deduplication = new DeliveryDeduplication(input.inbox);
     }
+    /**
+     * Notifies a fresh durable scan without starting a second shard session.
+     *
+     * @returns Whether the wake was recorded for a live shard run.
+     */
+    wake() {
+        if (!this.#reading)
+            return false;
+        this.#wakeGeneration += 1;
+        for (const target of this.#capacityBlockedTargets)
+            this.#blockedTargets.delete(target);
+        this.#capacityBlockedTargets.clear();
+        this.#wakeGate.resolve(undefined);
+        this.#wakeGate = Promise.withResolvers();
+        return true;
+    }
+    /**
+     * Stops new work and cancels entered Agent work for explicit shutdown.
+     *
+     * @param reason Cause of cancellation or failure.
+     */
+    stop(reason) {
+        this.#stopped = true;
+        this.#agentController.abort(reason);
+        this.#clearRenewal();
+        this.wake();
+    }
+    /**
+     * Serializes validation, renewal, and release against one current session snapshot.
+     * @typeParam T Result of the admitted shard operation.
+     * @param onOperation Current-session operation serialized with renewal and release.
+     * @returns Result of the admitted session operation.
+     */
+    #sessionOperation(onOperation) {
+        const attempt = this.#sessionSerial.then(onOperation);
+        this.#sessionSerial = attempt.then(() => undefined, () => undefined);
+        return attempt;
+    }
+    /**
+     * Schedules renewal before the issued lease expires while live work remains.
+     */
+    #scheduleRenewal() {
+        if (this.#stopped || this.#current.kind !== "LEASED" || this.input.shards.renew === undefined)
+            return;
+        this.#clearRenewal();
+        const life = this.#current.expiresAt.getTime() - this.#current.pickedUpAt.getTime();
+        const delay = Math.max(1, Math.min(Math.floor(life / 2), this.#current.expiresAt.getTime() - Time.currentTimeMillis() - 1));
+        this.#renewalTimer = setTimeout(() => {
+            void this.#renewLease();
+        }, delay);
+        this.#renewalTimer.unref();
+    }
+    /**
+     * Updates one live lease and fences the run permanently on loss.
+     * @returns Completion of lease renewal or loss fencing.
+     */
+    async #renewLease() {
+        await this.#sessionOperation(async () => {
+            const renew = this.input.shards.renew?.bind(this.input.shards);
+            if (this.#current.kind !== "LEASED" || this.#stopped || renew === undefined)
+                return;
+            const renewed = await safelyValue(() => renew(this.#current), undefined);
+            if (renewed === undefined)
+                this.stop(new Error("Shard ownership was lost."));
+            else
+                this.#current = renewed;
+        });
+        if (!this.#stopped && this.#liveAgents.size > 0)
+            this.#scheduleRenewal();
+    }
+    /**
+     * Cancels the live lease timer after completion or loss.
+     */
+    #clearRenewal() {
+        if (this.#renewalTimer !== undefined)
+            clearTimeout(this.#renewalTimer);
+        this.#renewalTimer = undefined;
+    }
     /**
      * Executes delivery and releases the shard session.
      *
@@ -243,10 +700,12 @@ class DeliveryDrain {
             run = (await this.#canStart()) ? await this.#readPages() : this.#complete("STOPPED");
         }
         catch (error) {
-            if (!(await this.#release()))
-                return this.#complete("FAILED");
+            this.#reading = false;
+            this.stop(error);
+            await this.#release();
             throw error;
         }
+        this.#reading = false;
         return (await this.#release()) ? run : this.#complete("FAILED");
     }
     /**
@@ -255,7 +714,20 @@ class DeliveryDrain {
      * @returns Whether the session was released.
      */
     async #release() {
-        const released = await safelyValue(() => this.input.shards.release(this.#current, this.input.options.operation), false);
+        if (this.#stopped || this.#ownershipLost || this.input.options.operation?.signal?.aborted)
+            this.#agentController.abort();
+        await Promise.allSettled(this.#liveAgents);
+        this.#clearRenewal();
+        let released = false;
+        try {
+            released = await this.#sessionOperation(() => this.input.shards.release(this.#current, this.input.options.operation));
+        }
+        catch (error) {
+            this.#releaseFailure =
+                error instanceof Error ? error : new Error("Shard release failed.", { cause: error });
+        }
+        if (!released && this.#releaseFailure === undefined)
+            this.#releaseFailure = new Error("Shard release was not confirmed.");
         if (released)
             await safely(() => this.#completeMonitoring());
         return released;
@@ -278,14 +750,19 @@ class DeliveryDrain {
     async #readPages() {
         let after;
         let deliveredInScan = false;
+        let scanGeneration = this.#wakeGeneration;
         for (;;) {
+            if (this.#stopped || this.#ownershipLost || this.input.options.operation?.signal?.aborted)
+                return this.#complete("STOPPED");
+            const wake = this.#wakeGate.promise;
             const messages = await this.#readPage(after);
             this.#removedInPage.clear();
             if (messages.length === 0) {
-                if (!deliveredInScan)
+                if (await this.#emptyPageDrained(deliveredInScan, scanGeneration, wake))
                     return this.#complete("DRAINED");
                 after = undefined;
                 deliveredInScan = false;
+                scanGeneration = this.#wakeGeneration;
                 continue;
             }
             const deliveredBefore = this.#statistics.delivered;
@@ -295,10 +772,11 @@ class DeliveryDrain {
                 return this.#complete("STOPPED");
             deliveredInScan ||= this.#statistics.delivered !== deliveredBefore;
             if (messages.length < this.input.pageSize) {
-                if (!deliveredInScan)
+                if (await this.#shortPageDrained(deliveredInScan, scanGeneration, wake))
                     return this.#complete("DRAINED");
                 after = undefined;
                 deliveredInScan = false;
+                scanGeneration = this.#wakeGeneration;
                 continue;
             }
             const survivor = messages.findLast((message) => !this.#removedInPage.has(message));
@@ -306,6 +784,51 @@ class DeliveryDrain {
                 after = this.#continuation([survivor]);
         }
     }
+    /**
+     * Waits for admitted Agent work after an empty Inbox page.
+     * @param deliveredInScan Whether the current scan delivered a row.
+     * @param generation Wake generation captured before reading the page.
+     * @param wake Notification that new work became ready.
+     * @returns Whether the empty page ends this drain.
+     */
+    async #emptyPageDrained(deliveredInScan, generation, wake) {
+        if (generation !== this.#wakeGeneration)
+            return false;
+        if (this.#liveAgents.size > 0) {
+            await Promise.race([...this.#liveAgents, wake]);
+            return false;
+        }
+        return !deliveredInScan && this.#endScan(generation);
+    }
+    /**
+     * Waits for live Agent work before deciding that a short page is drained.
+     * @param deliveredInScan Whether the current scan delivered a row.
+     * @param generation Wake generation captured before reading the page.
+     * @param wake Notification that new work became ready.
+     * @returns Whether the short page ends this drain.
+     */
+    async #shortPageDrained(deliveredInScan, generation, wake) {
+        if (deliveredInScan)
+            return false;
+        if (generation !== this.#wakeGeneration)
+            return false;
+        if (this.#liveAgents.size === 0)
+            return this.#endScan(generation);
+        await Promise.race([...this.#liveAgents, wake]);
+        return false;
+    }
+    /**
+     * Checks the terminal scan decision against a concurrent durable wake.
+     *
+     * @param generation Wake generation captured before reading the page.
+     * @returns Whether this scan may finish without losing a notification.
+     */
+    #endScan(generation) {
+        if (generation !== this.#wakeGeneration)
+            return false;
+        this.#reading = false;
+        return true;
+    }
     /**
      * Reads the next inbox page after an optional continuation.
      *
@@ -343,19 +866,27 @@ class DeliveryDrain {
      * @returns Whether processing may continue.
      */
     async #processMessage(message, page) {
-        if (this.input.options.operation?.signal?.aborted)
+        if (this.#stopped || this.input.options.operation?.signal?.aborted)
             return false;
         if (this.#shouldSkip(message))
             return true;
-        const target = this.#targetKey(message);
-        if (this.#blockedTargets.has(target))
-            return true;
-        this.#statistics.processed += 1;
-        if (!(await this.#mayProcess()))
-            return false;
-        if (page.isDuplicate(message))
-            return this.#removeDuplicate(message, target);
-        return this.#deliver(message, target);
+        try {
+            const target = this.#targetKey(message);
+            if (this.#blockedTargets.has(target))
+                return true;
+            this.#statistics.processed += 1;
+            if (!(await this.#mayProcess()))
+                return false;
+            if (page.isDuplicate(message)) {
+                const continued = await this.#removeDuplicate(message, target);
+                this.#reportOutcome(message);
+                return continued;
+            }
+            return await this.#deliver(message, target);
+        }
+        finally {
+            controlledMessageSettlers.get(this.input.options)?.(message);
+        }
     }
     /**
      * Determines whether an inbox message is ineligible for endpoint delivery.
@@ -375,7 +906,11 @@ class DeliveryDrain {
      * @returns The target type and ID key.
      */
     #targetKey(message) {
-        return `${message.inboxId.targetTypeUrl}:${InboxTargets.key(message.inboxId.targetId)}`;
+        return JSON.stringify([
+            deliveryTenantKey(message),
+            message.inboxId.targetTypeUrl,
+            InboxTargets.key(message.inboxId.targetId),
+        ]);
     }
     /**
      * Checks monitoring and verifies that the shard session remains current.
@@ -395,9 +930,10 @@ class DeliveryDrain {
      */
     async #removeDuplicate(message, target) {
         try {
-            if (!(await this.input.inbox.removeDuplicate(message, this.#current, this.input.options.operation)))
+            if (!(await this.input.inbox.removeDuplicate(message, this.input.options.operation)))
                 throw new Error("Inbox duplicate was not removed.");
             this.#removedInPage.add(message);
+            this.#messageOutcomes.set(message, { kind: "acknowledged", agent: false });
             this.input.options.onDuplicateRemoved?.(message);
         }
         catch (error) {
@@ -414,35 +950,156 @@ class DeliveryDrain {
      * @returns Whether processing may continue.
      */
     async #deliver(message, target) {
+        const acknowledgement = { completed: false, exhausted: false };
+        await this.#processAttempt(message, target, acknowledgement);
+        this.#reportOutcome(message);
+        return !this.#ownershipLost && !this.input.options.operation?.signal?.aborted;
+    }
+    /**
+     * Records one selected row after its endpoint and monitor policy settle.
+     * @param message Inbox row whose settled outcome is reported.
+     */
+    #reportOutcome(message) {
+        const outcome = this.#messageOutcomes.get(message);
+        if (outcome !== undefined)
+            this.input.options.onMessageOutcome?.(message, outcome);
+    }
+    /**
+     * Executes one dispatch attempt and the monitor-selected response to a receptor failure.
+     *
+     * @param message The persisted Inbox message.
+     * @param target The target key used to isolate acknowledgement failure.
+     * @param acknowledgement Shared status across immediate repeats.
+     * @returns Completion of the attempt and its selected action.
+     */
+    async #processAttempt(message, target, acknowledgement) {
+        const observation = receptionObservation.create();
+        const generation = this.#wakeGeneration;
         try {
+            const agent = await this.#dispatch(message, observation);
+            if (agent === dispatchNotAdmitted) {
+                acknowledgement.notAdmitted = true;
+                return;
+            }
             this.#statistics.accepted += 1;
-            await this.#dispatch(message);
-            if (this.input.options.operation?.signal?.aborted || !(await this.#validate()))
-                return false;
-            await this.#markDelivered(message);
+            if (agent?.kind === "deferred") {
+                this.#recordDeferred(message, target, generation);
+                return;
+            }
+            if (agent?.kind === "error") {
+                this.#recordFailure(message, agent.error);
+                this.#blockedTargets.add(target);
+                return;
+            }
+            if (agent?.kind === "accepted") {
+                await this.#completeAgentAttempt(message, target, acknowledgement, agent);
+                return;
+            }
         }
         catch (error) {
-            await this.#handleReceptionFailure(message, target, error);
-            if (this.#ownershipLost)
-                return false;
+            await this.#handleReceptionFailure(message, target, acknowledgement, observation, error);
+            if (this.#ownershipLost || this.input.options.operation?.signal?.aborted)
+                return;
         }
-        return true;
+        await this.#completeAttempt(message, target, acknowledgement);
     }
     /**
-     * Records endpoint failure and runs the configured recovery action.
-     *
-     * @param message Supplies the failed inbox message.
-     * @param target Identifies the message target.
-     * @param error Supplies the endpoint or acknowledgement failure.
-     * @returns Completion after the recovery action finishes.
+     * Completes acknowledgement before starting an admitted Agent handler.
+     * @param message Persisted Inbox row.
+     * @param target Canonical target key.
+     * @param acknowledgement Status shared with immediate repeats.
+     * @param agent Admitted handler awaiting durable acknowledgement.
+     * @returns Completion after acknowledgement and handler start or discard.
+     */
+    async #completeAgentAttempt(message, target, acknowledgement, agent) {
+        await this.#completeAttempt(message, target, acknowledgement);
+        if (acknowledgement.completed) {
+            this.#messageOutcomes.set(message, { kind: "acknowledged", agent: true });
+            this.#startAgent(message, agent);
+        }
+        else
+            agent.discard();
+    }
+    /**
+     * Records a capacity deferral and revisits it if release raced with dispatch.
+     * @param generation Wake generation captured before admission.
+     * @param message Inbox row deferred for Agent capacity.
+     * @param target Canonical target key blocked in this scan.
      */
-    async #handleReceptionFailure(message, target, error) {
+    #recordDeferred(message, target, generation) {
+        this.#messageOutcomes.set(message, { kind: "deferred" });
+        this.#blockedTargets.add(target);
+        this.#capacityBlockedTargets.add(target);
+        if (generation !== this.#wakeGeneration)
+            this.wake();
+    }
+    /**
+     * Applies the original receptor monitor only to an actual receptor failure.
+     * @param acknowledgement Status shared with repeated attempts.
+     * @param error Original reception failure.
+     * @param message Inbox row whose receptor failed.
+     * @param observation Whether the receptor callback was entered.
+     * @param target Canonical target key for acknowledgement.
+     * @returns Completion of the monitor-selected failure action.
+     */
+    async #handleReceptionFailure(message, target, acknowledgement, observation, error) {
+        if (this.#ownershipLost || this.input.options.operation?.signal?.aborted) {
+            this.#recordFailure(message, error);
+            return;
+        }
+        if (!receptionObservation.isReceptorFailure(observation, error))
+            throw error;
         this.#recordFailure(message, error);
-        const reception = new FailedReception(message, error, () => this.#acknowledge(message), () => this.#repeat(message));
-        const action = await safelyValue(() => this.input.monitor.onReceptionFailure(reception), reception.markDelivered());
-        if (!(await safely(() => action.execute())) &&
-            !(await safely(() => reception.markDelivered().execute())))
+        const reception = new FailedReception(message, error, () => this.#acknowledgeOnce(message, acknowledgement), () => this.#processAttempt(message, target, acknowledgement));
+        const action = await this.input.monitor.onReceptionFailure(reception);
+        await action.execute();
+    }
+    /**
+     * Marks completed processing once, including nested repeat recovery.
+     *
+     * @param message The processed Inbox message.
+     * @param target The target key used to isolate acknowledgement failure.
+     * @param acknowledgement Shared status across immediate repeats.
+     * @returns Completion after acknowledgement or exhausted retry containment.
+     */
+    async #completeAttempt(message, target, acknowledgement) {
+        if (acknowledgement.completed ||
+            acknowledgement.exhausted ||
+            acknowledgement.notAdmitted ||
+            this.input.options.operation?.signal?.aborted)
+            return;
+        if (!(await this.#validate()))
+            return;
+        try {
+            await this.#acknowledgeWithRetry(message, acknowledgement);
+        }
+        catch (error) {
+            acknowledgement.exhausted = true;
+            this.#recordFailure(message, error);
             this.#blockedTargets.add(target);
+        }
+    }
+    /**
+     * Retries only durable acknowledgement after processing has completed.
+     *
+     * @param message Supplies the successfully processed message.
+     * @param acknowledgement Shared status across immediate repeats.
+     * @returns Completion after acknowledgement.
+     */
+    async #acknowledgeWithRetry(message, acknowledgement) {
+        let firstError;
+        for (let attempt = 0; attempt < 3; attempt += 1) {
+            try {
+                await this.#acknowledgeOnce(message, acknowledgement);
+                return;
+            }
+            catch (error) {
+                firstError ??= error;
+                if (this.#ownershipLost || this.input.options.operation?.signal?.aborted)
+                    break;
+            }
+        }
+        throw firstError;
     }
     /**
      * Marks a message delivered after checking the shard session.
@@ -456,26 +1113,69 @@ class DeliveryDrain {
         await this.#markDelivered(message);
     }
     /**
-     * Dispatches the message again and acknowledges it.
+     * Marks a message delivered once across nested monitor-selected repeats.
      *
-     * @param message Supplies the inbox message to repeat.
-     * @returns Completion after dispatch and acknowledgement.
+     * @param message The message to mark delivered.
+     * @param acknowledgement Shared status across immediate repeats.
+     * @returns Completion after acknowledgement.
      */
-    async #repeat(message) {
-        await this.#dispatch(message);
+    async #acknowledgeOnce(message, acknowledgement) {
+        if (acknowledgement.completed)
+            return;
         await this.#acknowledge(message);
+        acknowledgement.completed = true;
     }
     /**
      * Dispatches a message while validating the shard session before the call.
      *
      * @param message Supplies the inbox message for the endpoint.
+     * @param observation Records failure origin for this attempt.
      * @returns Completion after endpoint dispatch.
      */
-    async #dispatch(message) {
-        await withDeliveryCommitFence(async () => {
+    async #dispatch(message, observation) {
+        const operation = this.input.options.operation?.signal;
+        const signal = operation === undefined
+            ? this.#agentController.signal
+            : AbortSignal.any([this.#agentController.signal, operation]);
+        const admission = { rejected: false };
+        const outcome = await AgentDeliveryScope.run(() => receptionObservation.run(observation, () => withDeliveryCommitFence(async () => {
             if (!(await this.#validate()))
                 throw new Error("Shard ownership was lost.");
-        }, () => Promise.resolve(this.input.options.onMessage(message)));
+        }, () => {
+            if (this.input.options.acceptMessage?.(message) === false) {
+                admission.rejected = true;
+                return Promise.resolve();
+            }
+            observation.entered = true;
+            return Promise.resolve(this.input.options.onMessage(message));
+        })), signal);
+        return admission.rejected ? dispatchNotAdmitted : outcome;
+    }
+    /**
+     * Records admitted Agent work under this shard session after Inbox acknowledgement.
+     * @param agent Accepted Agent turn and discard callback.
+     * @param message Acknowledged Inbox row.
+     */
+    #startAgent(message, agent) {
+        const operation = this.input.options.operation?.signal;
+        const signal = operation === undefined
+            ? this.#agentController.signal
+            : AbortSignal.any([this.#agentController.signal, operation]);
+        const checkSession = async () => {
+            if (!(await this.#validate()))
+                throw new Error("Shard ownership was lost.");
+        };
+        const work = Promise.resolve()
+            .then(() => agent.run(signal, checkSession))
+            .catch((error) => {
+            this.#recordFailure(message, error);
+        })
+            .finally(() => {
+            this.#liveAgents.delete(work);
+            this.wake();
+        });
+        this.#liveAgents.add(work);
+        this.#scheduleRenewal();
     }
     /**
      * Marks a message delivered and updates local counts and callbacks.
@@ -488,7 +1188,13 @@ class DeliveryDrain {
             throw new Error("Inbox message was not marked delivered.");
         this.#deduplication.recordDelivered(message);
         this.#statistics.delivered += 1;
-        this.input.options.onDelivered?.(message);
+        this.#messageOutcomes.set(message, { kind: "acknowledged", agent: false });
+        try {
+            this.input.options.onDelivered?.(message);
+        }
+        catch {
+            // A post-ack observer cannot undo durable delivery or repeat processing.
+        }
     }
     /**
      * Removes expired delivered messages from one page when supported.
@@ -525,9 +1231,7 @@ class DeliveryDrain {
             return false;
         if (this.input.options.operation?.signal?.aborted || !(await this.#validate()))
             return undefined;
-        const removed = await this.input.inbox.removeDelivered(message, this.#current, this.input.options.operation);
-        if (this.input.options.operation?.signal?.aborted || (!removed && !(await this.#validate())))
-            return undefined;
+        const removed = await this.input.inbox.removeDelivered(message, this.input.options.operation);
         return removed;
     }
     /**
@@ -558,12 +1262,18 @@ class DeliveryDrain {
      * @returns Whether the shard session remains valid.
      */
     async #validate() {
-        const current = await safelyValue(() => this.input.shards.validateOwnership(this.#current, this.input.options.operation), undefined);
-        if (current === undefined)
-            this.#ownershipLost = true;
-        else
-            this.#current = current;
-        return current !== undefined;
+        return this.#sessionOperation(async () => {
+            if (this.#stopped || this.#ownershipLost)
+                return false;
+            const current = await safelyValue(() => this.input.shards.validateOwnership(this.#current, this.input.options.operation), undefined);
+            if (current === undefined) {
+                this.#ownershipLost = true;
+                this.stop(new Error("Shard ownership was lost."));
+            }
+            else
+                this.#current = current;
+            return current !== undefined;
+        });
     }
     /**
      * Records an ephemeral delivery failure up to the configured limit.
@@ -573,6 +1283,8 @@ class DeliveryDrain {
      */
     #recordFailure(message, error) {
         this.#statistics.failed += 1;
+        if (this.#messageOutcomes.get(message)?.kind !== "acknowledged")
+            this.#messageOutcomes.set(message, { kind: "failed", error });
         if (this.#failures.length < deliveryFailureLimit)
             this.#failures.push(Object.freeze({ message: snapshot(message), error }));
         this.#failureLimitReached = this.#failures.length === deliveryFailureLimit;
@@ -584,6 +1296,7 @@ class DeliveryDrain {
      * @returns The immutable delivery result.
      */
     #complete(status) {
+        this.#reading = false;
         return result(status, this.#statistics, this.#failures);
     }
     /**
@@ -774,6 +1487,7 @@ class RecentDeliveries {
      */
     static key(message) {
         return JSON.stringify([
+            deliveryTenantKey(message),
             message.signalId,
             message.inboxId.targetTypeUrl,
             InboxTargets.key(message.inboxId.targetId),