@spine-event-engine/server 2.0.0-snapshot.22 → 2.0.0-snapshot.24

package/dist/server/environment-delivery-worker.js

@@ -19,6 +19,7 @@ import { CommandSchema, EventSchema } from "@spine-event-engine/proto";
 import { deliveryRunWorkers, } from "../delivery/delivery-run-coordinator.js";
 import { DeliveryBuilder, UniformAcrossAllShards } from "../delivery/delivery-builder.js";
 import { Delivery, } from "../delivery/delivery.js";
+import { receptionObservation } from "../delivery/reception-observation.js";
 import { DeliverySupervisor, deliverySupervisorAccess, } from "../delivery/delivery-supervisor.js";
 import { DeliveryWorker } from "../delivery/delivery-worker.js";
 import { ShardIndex } from "../delivery/shard-index.js";
@@ -214,6 +215,12 @@ export class EnvironmentDeliveryWorker {
         const failures = retired.flatMap((result) => result.status === "rejected" ? [result.reason] : []);
         EnvironmentDeliveryValues.throwFailures(failures, "Environment registration worker retirement failed.");
     }
+    /**
+     * Finds the configured worker for an owner key.
+     *
+     * @param key The worker or group key.
+     * @returns The result of this delivery operation.
+     */
     #requiredWorker(key) {
         const worker = this.#workers.get(key);
         if (worker === undefined) {
@@ -221,6 +228,13 @@ export class EnvironmentDeliveryWorker {
         }
         return worker;
     }
+    /**
+     * Stops one configured owner worker and collects failures.
+     *
+     * @param key The worker or group key.
+     * @param worker The runtime delivery worker.
+     * @param failures The failures to propagate.
+     */
     #stopOwner(key, worker, failures) {
         if (!this.#stoppedWorkers.has(key)) {
             try {
@@ -239,14 +253,28 @@ export class EnvironmentDeliveryWorker {
         }
     }
 }
+/**
+ * Coordinates supervisor groups for one environment generation.
+ */
 class RuntimeDeliverySupervisor {
     #groups = new Map();
     #ports;
     #nodeId;
+    /**
+     * Creates the delivery component with its configured dependencies.
+     *
+     * @param ports The optional delivery ports.
+     * @param nodeId The worker node identity.
+     */
     constructor(ports, nodeId) {
         this.#ports = ports;
         this.#nodeId = nodeId;
     }
+    /**
+     * Adds the configured delivery runtime or shards.
+     *
+     * @param runtime The configured context delivery runtime.
+     */
     add(runtime) {
         const grouped = new Map();
         for (const shard of EnvironmentDeliveryValues.uniqueShards(runtime.scopes)) {
@@ -267,10 +295,20 @@ class RuntimeDeliverySupervisor {
             group.add(runtime.owner.key, runtime, shards);
         }
     }
+    /**
+     * Removes an owner route from this runtime.
+     *
+     * @param ownerKey The configured owner key.
+     */
     remove(ownerKey) {
         for (const group of this.#groups.values())
             group.remove(ownerKey);
     }
+    /**
+     * Starts the configured delivery worker or supervisor.
+     *
+     * @returns The result of this delivery operation.
+     */
     async start() {
         if (!managedDeliveryActivated) {
             waitingManagedSupervisors.add(() => {
@@ -282,16 +320,39 @@ class RuntimeDeliverySupervisor {
             return;
         await Promise.all(Array.from(this.#groups.values(), (group) => group.start()));
     }
+    /**
+     * Notifies the selected delivery scope or shard.
+     *
+     * @param ownerKey The configured owner key.
+     * @param shard The shard to process.
+     */
     notify(ownerKey, shard) {
         this.#groups.get(this.#groupKey(ownerKey, shard.ofTotal))?.notify(shard);
     }
+    /**
+     * Marks the selected owners as unavailable for new route admission.
+     *
+     * @param ownerKeys The owner keys in this generation.
+     */
     fenceOwners(ownerKeys) {
         for (const group of this.#groups.values())
             group.fenceOwners(ownerKeys);
     }
+    /**
+     * Waits for admitted work for the selected owners to settle.
+     *
+     * @param ownerKeys The owner keys in this generation.
+     * @returns The result of this delivery operation.
+     */
     async awaitOwnersSettled(ownerKeys) {
         await Promise.all(Array.from(this.#groups.values(), (group) => group.awaitOwnersSettled(ownerKeys)));
     }
+    /**
+     * Closes the selected owners after admitted work settles.
+     *
+     * @param ownerKeys The owner keys in this generation.
+     * @returns The result of this delivery operation.
+     */
     async retireOwners(ownerKeys) {
         for (const [key, group] of this.#groups) {
             await group.retireOwners(ownerKeys);
@@ -299,6 +360,9 @@ class RuntimeDeliverySupervisor {
                 this.#groups.delete(key);
         }
     }
+    /**
+     * Stops admission and starts delivery shutdown.
+     */
     stop() {
         const failures = [];
         for (const group of this.#groups.values()) {
@@ -311,20 +375,40 @@ class RuntimeDeliverySupervisor {
         }
         EnvironmentDeliveryValues.throwFailures(failures, "Environment delivery supervisor stop failed.");
     }
+    /**
+     * Waits for the active delivery work to settle.
+     *
+     * @returns The result of this delivery operation.
+     */
     async awaitSettled() {
         const settled = await Promise.allSettled(Array.from(this.#groups.values(), (group) => group.awaitSettled()));
         EnvironmentDeliveryValues.throwFailures(settled.flatMap((result) => (result.status === "rejected" ? [result.reason] : [])), "Environment delivery supervisor settlement failed.");
     }
+    /**
+     * Closes this delivery worker after settling its work.
+     *
+     * @returns The result of this delivery operation.
+     */
     async retire() {
         const settled = await Promise.allSettled(Array.from(this.#groups.values(), (group) => EnvironmentDeliveryValues.attempt(() => group.retire())));
         EnvironmentDeliveryValues.throwFailures(settled.flatMap((result) => (result.status === "rejected" ? [result.reason] : [])), "Environment delivery supervisor retirement failed.");
     }
+    /**
+     * Builds a stable supervisor-group key for one owner and shard total.
+     *
+     * @param ownerKey The configured owner key.
+     * @param shardCount The shard total for this supervisor group.
+     * @returns The result of this delivery operation.
+     */
     #groupKey(ownerKey, shardCount) {
         return this.#ports?.source === undefined
             ? `${ownerKey}/${String(shardCount)}`
             : `remote/${String(shardCount)}`;
     }
 }
+/**
+ * Keeps admitted routes stable while one shard supervisor drains.
+ */
 class RuntimeDeliverySupervisorGroup {
     #source;
     #supervisor;
@@ -336,6 +420,11 @@ class RuntimeDeliverySupervisorGroup {
     #start;
     #stopped = false;
     #close;
+    /**
+     * Creates the delivery component with its configured dependencies.
+     *
+     * @param options The delivery or cancellation options.
+     */
     constructor(options) {
         // `createSupervisorGroup()` rejects an empty shard collection before internal construction.
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@@ -352,6 +441,9 @@ class RuntimeDeliverySupervisorGroup {
             onMessage: (message) => this.#route(message),
         });
         deliverySupervisorAccess.installAdmission(this.#supervisor, (message) => this.#accept(message));
+        deliverySupervisorAccess.installMessageSettled(this.#supervisor, (message) => {
+            this.#settleReservation(message);
+        });
         deliverySupervisorAccess.installFinalization(this.#supervisor, (shard) => {
             this.#releaseUndeliveredReservations(shard);
         });
@@ -359,6 +451,13 @@ class RuntimeDeliverySupervisorGroup {
             deliverySupervisorAccess.installLogger(this.#supervisor, options.logger);
         }
     }
+    /**
+     * Adds the configured delivery runtime or shards.
+     *
+     * @param ownerKey The configured owner key.
+     * @param runtime The configured context delivery runtime.
+     * @param shards The shards assigned to this runtime.
+     */
     add(ownerKey, runtime, shards) {
         if (this.#source instanceof LocalDeliverySource)
             this.#source.add(shards);
@@ -378,13 +477,29 @@ class RuntimeDeliverySupervisorGroup {
             this.#routes.set(key, routes);
         }
     }
+    /**
+     * Checks whether this supervisor group has configured routes.
+     *
+     * @returns The result of this delivery operation.
+     */
     get empty() {
         return this.#routes.size === 0;
     }
+    /**
+     * Marks the selected owners as unavailable for new route admission.
+     *
+     * @param ownerKeys The owner keys in this generation.
+     */
     fenceOwners(ownerKeys) {
         for (const ownerKey of ownerKeys)
             this.#retiringOwners.add(ownerKey);
     }
+    /**
+     * Waits for admitted work for the selected owners to settle.
+     *
+     * @param ownerKeys The owner keys in this generation.
+     * @returns The result of this delivery operation.
+     */
     async awaitOwnersSettled(ownerKeys) {
         await Promise.all(Array.from(this.#reserved.values(), async (reservations) => Promise.all(Array.from(reservations.values())
             .filter((reservation) => ownerKeys.includes(reservation.route.ownerKey))
@@ -395,6 +510,12 @@ class RuntimeDeliverySupervisorGroup {
                 await Promise.allSettled(active);
         }));
     }
+    /**
+     * Closes the selected owners after admitted work settles.
+     *
+     * @param ownerKeys The owner keys in this generation.
+     * @returns The result of this delivery operation.
+     */
     async retireOwners(ownerKeys) {
         this.fenceOwners(ownerKeys);
         await this.awaitOwnersSettled(ownerKeys);
@@ -403,6 +524,11 @@ class RuntimeDeliverySupervisorGroup {
         if (this.empty)
             await this.retire();
     }
+    /**
+     * Removes an owner route from this runtime.
+     *
+     * @param ownerKey The configured owner key.
+     */
     remove(ownerKey) {
         for (const [key, routes] of this.#routes) {
             const retained = routes.filter((route) => route.ownerKey !== ownerKey);
@@ -412,6 +538,11 @@ class RuntimeDeliverySupervisorGroup {
                 this.#routes.set(key, retained);
         }
     }
+    /**
+     * Starts the configured delivery worker or supervisor.
+     *
+     * @returns The result of this delivery operation.
+     */
     start() {
         if (this.#stopped) {
             return Promise.reject(new Error("Environment delivery supervisor is stopped."));
@@ -424,6 +555,11 @@ class RuntimeDeliverySupervisorGroup {
         }
         return this.#start;
     }
+    /**
+     * Notifies the selected delivery scope or shard.
+     *
+     * @param shard The shard to process.
+     */
     notify(shard) {
         if (this.#stopped)
             return;
@@ -434,6 +570,9 @@ class RuntimeDeliverySupervisorGroup {
         })
             .catch(() => undefined);
     }
+    /**
+     * Stops admission and starts delivery shutdown.
+     */
     stop() {
         if (this.#stopped)
             return;
@@ -442,28 +581,54 @@ class RuntimeDeliverySupervisorGroup {
         // spine-log-boundary: server.delivery_supervisor_close
         void this.#close.catch(() => undefined);
     }
+    /**
+     * Waits for the active delivery work to settle.
+     *
+     * @returns The result of this delivery operation.
+     */
     async awaitSettled() {
         await this.#start;
         await this.#close;
     }
+    /**
+     * Closes this delivery worker after settling its work.
+     *
+     * @returns The result of this delivery operation.
+     */
     async retire() {
         this.stop();
         await this.awaitSettled();
     }
+    /**
+     * Accepts an Inbox message and reserves its exact route.
+     *
+     * @param message The persisted Inbox message.
+     * @returns The result of this delivery operation.
+     */
     #accept(message) {
         const route = this.#select(message);
         if (route !== undefined) {
             // `options.shards` initializes one private reservation map per supervised shard.
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
             const reservations = this.#reserved.get(message.shard.key());
-            reservations.set(_a.messageKey(message), {
+            const key = _a.messageKey(message);
+            if (reservations.has(key))
+                return false;
+            reservations.set(key, {
                 route,
                 settled: Promise.withResolvers(),
             });
         }
         return route !== undefined;
     }
+    /**
+     * Routes an admitted Inbox message to its reserved endpoint.
+     *
+     * @param message The persisted Inbox message.
+     * @returns The result of this delivery operation.
+     */
     #route(message) {
+        receptionObservation.markManaged();
         const key = _a.messageKey(message);
         // `Delivery` invokes the callback only after this group's admission reserved its route.
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@@ -481,10 +646,29 @@ class RuntimeDeliverySupervisorGroup {
                 this.#active.delete(route.ownerKey);
         });
         active.add(replay);
+        return replay;
+    }
+    /**
+     * Resolves and removes one reservation after its complete processing action.
+     *
+     * @param message The settled Inbox message.
+     */
+    #settleReservation(message) {
+        // The constructor initializes a reservation map for every supervised shard.
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const reservations = this.#reserved.get(message.shard.key());
+        const key = _a.messageKey(message);
+        const reservation = reservations.get(key);
+        if (reservation === undefined)
+            return;
         reservations.delete(key);
         reservation.settled.resolve(undefined);
-        return replay;
     }
+    /**
+     * Clears route reservations when a shard run finishes.
+     *
+     * @param shard The shard to process.
+     */
     #releaseUndeliveredReservations(shard) {
         // `options.shards` initializes one private reservation map per supervised shard.
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@@ -494,12 +678,25 @@ class RuntimeDeliverySupervisorGroup {
         }
         reservations.clear();
     }
+    /**
+     * Finds the configured route for an Inbox message.
+     *
+     * @param message The persisted Inbox message.
+     * @returns The result of this delivery operation.
+     */
     #select(message) {
         const key = _a.key(message);
         const routes = this.#routes.get(key);
         const route = _a.select(routes, message);
         return route === undefined || this.#retiringOwners.has(route.ownerKey) ? undefined : route;
     }
+    /**
+     * Finds the matching route for an Inbox message.
+     *
+     * @param routes The candidate routes.
+     * @param message The persisted Inbox message.
+     * @returns The result of this delivery operation.
+     */
     static select(routes, message) {
         if (routes === undefined || routes.length === 0)
             return undefined;
@@ -508,7 +705,7 @@ class RuntimeDeliverySupervisorGroup {
     }
     /**
      *
-     * Derives a tenant boundary key from the delivery label's matching envelope.
+     * Finds the tenant boundary key from the delivery label's matching envelope.
      * `HANDLE_COMMAND` reads the Command actor tenant; Event labels read imported
      * or past-message Event origin tenants.
      *
@@ -539,30 +736,63 @@ class RuntimeDeliverySupervisorGroup {
         }
         return undefined;
     }
+    /**
+     * Builds a route key for one configured endpoint.
+     *
+     * @param route The configured route.
+     * @returns The result of this delivery operation.
+     */
     static key(route) {
         const targetTypeUrl = route.inboxId?.targetTypeUrl ?? route.targetTypeUrl;
         if (targetTypeUrl === undefined)
             throw new Error("Environment delivery route has no target type.");
         return JSON.stringify([route.label, targetTypeUrl, route.shard.index, route.shard.ofTotal]);
     }
+    /**
+     * Builds an exact route-reservation key for one Inbox message.
+     *
+     * @param message The persisted Inbox message.
+     * @returns The result of this delivery operation.
+     */
     static messageKey(message) {
-        return `${message.signalId}/${_a.key(message)}`;
+        return `${message.id.shard.key()}/${message.id.value}`;
     }
 }
 _a = RuntimeDeliverySupervisorGroup;
+/**
+ * Supplies local shard recovery observations to the supervisor.
+ */
 class LocalDeliverySource {
     #shards = new Map();
     #recoveryEnabled = false;
+    /**
+     * Creates the delivery component with its configured dependencies.
+     *
+     * @param shards The shards assigned to this runtime.
+     */
     constructor(shards) {
         this.add(shards);
     }
+    /**
+     * Adds the configured delivery runtime or shards.
+     *
+     * @param shards The shards assigned to this runtime.
+     */
     add(shards) {
         for (const shard of shards)
             this.#shards.set(shard.key(), shard);
     }
+    /**
+     * Sets local shards as eligible for periodic recovery.
+     */
     enableRecovery() {
         this.#recoveryEnabled = true;
     }
+    /**
+     * Returns the locally recoverable shard snapshot.
+     *
+     * @returns The result of this delivery operation.
+     */
     shardSnapshot() {
         return Promise.resolve(this.#recoveryEnabled
             ? Array.from(this.#shards.values(), (shard) => ({
@@ -572,18 +802,36 @@ class LocalDeliverySource {
             }))
             : []);
     }
+    /**
+     * Observes shard changes until the operation is cancelled.
+     *
+     * @param options The delivery or cancellation options.
+     * @returns The result of this delivery operation.
+     */
     observeShardUpdates(options) {
         return EnvironmentDeliveryValues.updatesUntilAborted(options?.signal);
     }
+    /**
+     * Returns expired local ownership records.
+     *
+     * @returns The result of this delivery operation.
+     */
     releaseExpired() {
         return Promise.resolve([]);
     }
 }
 /**
+ * Groups private delivery-runtime assembly and failure operations.
  *
- * @internal Groups private delivery-runtime assembly and failure operations.
+ * @internal
  */
 const EnvironmentDeliveryValues = Object.freeze({
+    /**
+     * Validates a delivery source before use.
+     *
+     * @param source The source to validate.
+     * @returns The validated delivery source.
+     */
     requireSource(source) {
         if (source === null ||
             typeof source !== "object" ||
@@ -594,6 +842,14 @@ const EnvironmentDeliveryValues = Object.freeze({
         }
         return source;
     },
+    /**
+     * Creates the worker that drains one context runtime.
+     *
+     * @param runtime The context runtime to serve.
+     * @param ports Optional delivery ports.
+     * @param nodeId Optional node identity.
+     * @returns The configured worker.
+     */
     createWorker(runtime, ports, nodeId) {
         const delivery = new Delivery({
             context: runtime.context,
@@ -608,6 +864,15 @@ const EnvironmentDeliveryValues = Object.freeze({
         });
         return deliveryRunWorkers.worker(worker);
     },
+    /**
+     * Creates a supervisor group for shards with one total.
+     *
+     * @param runtime The context runtime to serve.
+     * @param shards The assigned shards.
+     * @param ports Optional delivery ports.
+     * @param nodeId Optional node identity.
+     * @returns The configured supervisor group.
+     */
     createSupervisorGroup(runtime, shards, ports, nodeId) {
         const first = shards[0];
         if (first === undefined) {
@@ -627,8 +892,19 @@ const EnvironmentDeliveryValues = Object.freeze({
             ...(runtime.logger === undefined ? {} : { logger: runtime.logger }),
         });
     },
+    /**
+     * Creates an update stream that finishes on cancellation.
+     *
+     * @param signal The optional cancellation signal.
+     * @returns The cancellation-aware stream.
+     */
     updatesUntilAborted(signal) {
         return {
+            /**
+             * Returns the cancellation-aware update iterator.
+             *
+             * @returns The result of this delivery operation.
+             */
             [Symbol.asyncIterator]() {
                 return {
                     next: () => new Promise((resolve) => {
@@ -644,6 +920,12 @@ const EnvironmentDeliveryValues = Object.freeze({
             },
         };
     },
+    /**
+     * Finds the sole owner shared by the supplied scopes.
+     *
+     * @param scopes The exact delivery scopes.
+     * @returns Their shared owner.
+     */
     soleOwner(scopes) {
         const first = scopes[0]?.owner;
         if (first === undefined || scopes.some(({ owner }) => owner.key !== first.key)) {
@@ -651,6 +933,12 @@ const EnvironmentDeliveryValues = Object.freeze({
         }
         return first;
     },
+    /**
+     * Collects distinct shards from the supplied scopes.
+     *
+     * @param scopes The exact delivery scopes.
+     * @returns Their distinct shards.
+     */
     uniqueShards(scopes) {
         const shards = new Map();
         for (const { ready } of scopes) {
@@ -658,6 +946,12 @@ const EnvironmentDeliveryValues = Object.freeze({
         }
         return Object.freeze([...shards.values()]);
     },
+    /**
+     * Throws collected failures without losing their identity.
+     *
+     * @param failures The failures to propagate.
+     * @param message The aggregate message when several failures occurred.
+     */
     throwFailures(failures, message) {
         if (failures.length === 1) {
             throw failures[0];
@@ -666,6 +960,12 @@ const EnvironmentDeliveryValues = Object.freeze({
             throw new AggregateError(failures, message);
         }
     },
+    /**
+     * Awaits one asynchronous cleanup action.
+     *
+     * @param action The action to await.
+     * @returns Completion of that action.
+     */
     async attempt(action) {
         await action();
     },