@spine-event-engine/server 2.0.0-snapshot.11

package/dist/server/environment-delivery-worker.js

@@ -0,0 +1,673 @@
+/*
+ * Copyright 2026, CodeMatters. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+var _a;
+import {} from "@spine-event-engine/storage";
+import { TenantBoundary } from "@spine-event-engine/storage/provider";
+import { fromBinary } from "@bufbuild/protobuf";
+import { CommandSchema, EventSchema } from "@spine-event-engine/proto";
+import { deliveryRunWorkers, } from "../delivery/delivery-run-coordinator.js";
+import { DeliveryBuilder, UniformAcrossAllShards } from "../delivery/delivery-builder.js";
+import { Delivery, } from "../delivery/delivery.js";
+import { DeliverySupervisor, deliverySupervisorAccess, } from "../delivery/delivery-supervisor.js";
+import { DeliveryWorker } from "../delivery/delivery-worker.js";
+import { ShardIndex } from "../delivery/shard-index.js";
+const managedChild = process.env.SPINE_MANAGED_SERVER_CHILD === "true";
+let cancelManagedDelivery = false;
+let managedDeliveryActivated = !managedChild;
+const waitingManagedSupervisors = new Set();
+/**
+ * Controls private managed-child Delivery admission.
+ *
+ * @internal
+ */
+export const environmentDeliveryWorkerAccess = Object.freeze({
+    activateManagedChild() {
+        managedDeliveryActivated = true;
+        for (const start of waitingManagedSupervisors)
+            start();
+        waitingManagedSupervisors.clear();
+    },
+    cancelManagedChild() {
+        cancelManagedDelivery = true;
+        waitingManagedSupervisors.clear();
+    },
+});
+/**
+ * Routes each owner-partitioned obligation to its exact runtime worker.
+ *
+ * @internal
+ */
+export class EnvironmentDeliveryWorker {
+    #workers = new Map();
+    #supervisor;
+    #stoppedWorkers = new Set();
+    #stoppedSupervisors = new Set();
+    #stoppedOwners = new Set();
+    #createWorker;
+    #nodeId;
+    #ports;
+    /**
+     * Creates an environment delivery worker.
+     *
+     * @param options Optionally supplies a worker factory for lifecycle tests.
+     */
+    constructor(options = {}) {
+        this.#createWorker = options.createWorker ?? EnvironmentDeliveryValues.createWorker;
+        this.#ports = options.ports;
+        this.#nodeId = options.nodeId;
+        this.#supervisor = new RuntimeDeliverySupervisor(this.#ports, this.#nodeId);
+    }
+    /**
+     * Adds one owner-specific delivery runtime.
+     * @param runtime Supplies the runtime to configure.
+     */
+    add(runtime) {
+        if (this.#workers.has(runtime.owner.key)) {
+            throw new Error("Environment delivery owner is already configured.");
+        }
+        const worker = this.#createWorker(runtime, this.#ports, this.#nodeId);
+        this.#workers.set(runtime.owner.key, worker);
+        try {
+            this.#supervisor.add(runtime);
+        }
+        catch (error) {
+            this.#workers.delete(runtime.owner.key);
+            throw error;
+        }
+        void this.#supervisor.start();
+    }
+    /**
+     * Starts selected shards for one owner-specific obligation.
+     *
+     * @param obligation Supplies the finite delivery obligation.
+     * @param shards Selects shards to run.
+     * @returns Evidence produced by the worker.
+     */
+    start(obligation, shards) {
+        const owner = EnvironmentDeliveryValues.soleOwner(obligation.scopes);
+        const worker = this.#workers.get(owner.key);
+        if (worker === undefined) {
+            return Promise.reject(new Error("Environment delivery owner is not configured."));
+        }
+        return worker.start(obligation, shards).then(async (evidence) => {
+            await this.#supervisor.start();
+            return evidence;
+        });
+    }
+    /**
+     * Notifies a running supervisor about new work.
+     * @param scope Identifies the changed scope.
+     */
+    notify(scope) {
+        this.#requiredWorker(scope.owner.key);
+        this.#supervisor.notify(scope.owner.key, scope.ready.shard);
+    }
+    /**
+     * Stops every configured owner and aggregates failures.
+     */
+    stop() {
+        const failures = [];
+        for (const [key, worker] of this.#workers) {
+            if (this.#stoppedOwners.has(key)) {
+                continue;
+            }
+            this.#stopOwner(key, worker, failures);
+        }
+        try {
+            this.#supervisor.stop();
+        }
+        catch (error) {
+            failures.push(error);
+        }
+        EnvironmentDeliveryValues.throwFailures(failures, "Environment delivery worker stop failed.");
+    }
+    /**
+     * Awaits every configured worker and supervisor.
+     *
+     * @returns A promise that resolves after all configured work settles.
+     */
+    async awaitSettled() {
+        await Promise.all([
+            ...Array.from(this.#workers.values(), (worker) => worker.awaitSettled()),
+            this.#supervisor.awaitSettled(),
+        ]);
+    }
+    /**
+     * Closes every configured worker and supervisor.
+     *
+     * @returns A promise that settles after all configured resources retire.
+     */
+    async retire() {
+        const settled = await Promise.allSettled([
+            ...Array.from(this.#workers.values(), (worker) => EnvironmentDeliveryValues.attempt(() => worker.retire())),
+            EnvironmentDeliveryValues.attempt(() => this.#supervisor.retire()),
+        ]);
+        const failures = [];
+        for (const result of settled) {
+            if (result.status === "rejected") {
+                failures.push(result.reason);
+            }
+        }
+        EnvironmentDeliveryValues.throwFailures(failures, "Environment delivery worker retirement failed.");
+    }
+    /**
+     * Stops selected owners and aggregates failures.
+     * @param ownerKeys Identifies owners to stop.
+     */
+    stopOwners(ownerKeys) {
+        const selected = ownerKeys.map((key) => {
+            const worker = this.#workers.get(key);
+            if (worker === undefined) {
+                throw new Error("Environment delivery owner is not configured.");
+            }
+            return { key, worker };
+        });
+        const failures = [];
+        this.#supervisor.fenceOwners(ownerKeys);
+        for (const { key, worker } of selected) {
+            if (this.#stoppedOwners.has(key)) {
+                continue;
+            }
+            this.#stopOwner(key, worker, failures);
+        }
+        EnvironmentDeliveryValues.throwFailures(failures, "Environment registration worker stop failed.");
+    }
+    /**
+     * Awaits selected owners.
+     *
+     * @param ownerKeys Identifies owners to await.
+     * @returns A promise that resolves after selected owner work settles.
+     */
+    async awaitOwnersSettled(ownerKeys) {
+        await Promise.all([
+            ...ownerKeys.map((key) => this.#requiredWorker(key).awaitSettled()),
+            this.#supervisor.awaitOwnersSettled(ownerKeys),
+        ]);
+    }
+    /**
+     * Closes selected owners and releases local state.
+     *
+     * @param ownerKeys Identifies owners to retire.
+     * @returns A promise that settles after selected owners retire.
+     */
+    async retireOwners(ownerKeys) {
+        const selected = ownerKeys.map((key) => ({ key, worker: this.#requiredWorker(key) }));
+        const retired = await Promise.allSettled(selected.map(({ worker }) => EnvironmentDeliveryValues.attempt(() => worker.retire())));
+        await this.#supervisor.retireOwners(ownerKeys);
+        for (const { key } of selected) {
+            this.#workers.delete(key);
+            this.#stoppedWorkers.delete(key);
+            this.#stoppedSupervisors.delete(key);
+            this.#stoppedOwners.delete(key);
+        }
+        const failures = retired.flatMap((result) => result.status === "rejected" ? [result.reason] : []);
+        EnvironmentDeliveryValues.throwFailures(failures, "Environment registration worker retirement failed.");
+    }
+    #requiredWorker(key) {
+        const worker = this.#workers.get(key);
+        if (worker === undefined) {
+            throw new Error("Environment delivery owner is not configured.");
+        }
+        return worker;
+    }
+    #stopOwner(key, worker, failures) {
+        if (!this.#stoppedWorkers.has(key)) {
+            try {
+                worker.stop();
+                this.#stoppedWorkers.add(key);
+            }
+            catch (error) {
+                failures.push(error);
+            }
+        }
+        if (!this.#stoppedSupervisors.has(key)) {
+            this.#stoppedSupervisors.add(key);
+        }
+        if (this.#stoppedWorkers.has(key) && this.#stoppedSupervisors.has(key)) {
+            this.#stoppedOwners.add(key);
+        }
+    }
+}
+class RuntimeDeliverySupervisor {
+    #groups = new Map();
+    #ports;
+    #nodeId;
+    constructor(ports, nodeId) {
+        this.#ports = ports;
+        this.#nodeId = nodeId;
+    }
+    add(runtime) {
+        const grouped = new Map();
+        for (const shard of EnvironmentDeliveryValues.uniqueShards(runtime.scopes)) {
+            const shards = grouped.get(shard.ofTotal) ?? [];
+            shards.push(shard);
+            grouped.set(shard.ofTotal, shards);
+        }
+        if (grouped.size === 0) {
+            throw new Error("Environment delivery supervisor requires at least one shard.");
+        }
+        for (const [shardCount, shards] of grouped) {
+            const key = this.#groupKey(runtime.owner.key, shardCount);
+            let group = this.#groups.get(key);
+            if (group === undefined) {
+                group = EnvironmentDeliveryValues.createSupervisorGroup(runtime, shards, this.#ports, this.#nodeId);
+                this.#groups.set(key, group);
+            }
+            group.add(runtime.owner.key, runtime, shards);
+        }
+    }
+    remove(ownerKey) {
+        for (const group of this.#groups.values())
+            group.remove(ownerKey);
+    }
+    async start() {
+        if (!managedDeliveryActivated) {
+            waitingManagedSupervisors.add(() => {
+                void this.start();
+            });
+            return;
+        }
+        if (cancelManagedDelivery)
+            return;
+        await Promise.all(Array.from(this.#groups.values(), (group) => group.start()));
+    }
+    notify(ownerKey, shard) {
+        this.#groups.get(this.#groupKey(ownerKey, shard.ofTotal))?.notify(shard);
+    }
+    fenceOwners(ownerKeys) {
+        for (const group of this.#groups.values())
+            group.fenceOwners(ownerKeys);
+    }
+    async awaitOwnersSettled(ownerKeys) {
+        await Promise.all(Array.from(this.#groups.values(), (group) => group.awaitOwnersSettled(ownerKeys)));
+    }
+    async retireOwners(ownerKeys) {
+        for (const [key, group] of this.#groups) {
+            await group.retireOwners(ownerKeys);
+            if (group.empty)
+                this.#groups.delete(key);
+        }
+    }
+    stop() {
+        const failures = [];
+        for (const group of this.#groups.values()) {
+            try {
+                group.stop();
+            }
+            catch (error) {
+                failures.push(error);
+            }
+        }
+        EnvironmentDeliveryValues.throwFailures(failures, "Environment delivery supervisor stop failed.");
+    }
+    async awaitSettled() {
+        const settled = await Promise.allSettled(Array.from(this.#groups.values(), (group) => group.awaitSettled()));
+        EnvironmentDeliveryValues.throwFailures(settled.flatMap((result) => (result.status === "rejected" ? [result.reason] : [])), "Environment delivery supervisor settlement failed.");
+    }
+    async retire() {
+        const settled = await Promise.allSettled(Array.from(this.#groups.values(), (group) => EnvironmentDeliveryValues.attempt(() => group.retire())));
+        EnvironmentDeliveryValues.throwFailures(settled.flatMap((result) => (result.status === "rejected" ? [result.reason] : [])), "Environment delivery supervisor retirement failed.");
+    }
+    #groupKey(ownerKey, shardCount) {
+        return this.#ports?.source === undefined
+            ? `${ownerKey}/${String(shardCount)}`
+            : `remote/${String(shardCount)}`;
+    }
+}
+class RuntimeDeliverySupervisorGroup {
+    #source;
+    #supervisor;
+    #routes = new Map();
+    #retiringOwners = new Set();
+    #active = new Map();
+    #reserved;
+    shardCount;
+    #start;
+    #stopped = false;
+    #close;
+    constructor(options) {
+        // `createSupervisorGroup()` rejects an empty shard collection before internal construction.
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const first = options.shards[0];
+        this.shardCount = first.ofTotal;
+        this.#reserved = new Map(options.shards.map((shard) => [shard.key(), new Map()]));
+        this.#source =
+            options.source === undefined
+                ? new LocalDeliverySource(options.shards)
+                : EnvironmentDeliveryValues.requireSource(options.source);
+        this.#supervisor = new DeliverySupervisor({
+            source: this.#source,
+            delivery: options.delivery,
+            onMessage: (message) => this.#route(message),
+        });
+        deliverySupervisorAccess.installAdmission(this.#supervisor, (message) => this.#accept(message));
+        deliverySupervisorAccess.installFinalization(this.#supervisor, (shard) => {
+            this.#releaseUndeliveredReservations(shard);
+        });
+        if (options.logger !== undefined) {
+            deliverySupervisorAccess.installLogger(this.#supervisor, options.logger);
+        }
+    }
+    add(ownerKey, runtime, shards) {
+        if (this.#source instanceof LocalDeliverySource)
+            this.#source.add(shards);
+        for (const scope of runtime.scopes) {
+            if (scope.ready.shard.ofTotal !== this.shardCount)
+                continue;
+            const key = _a.key(scope.ready);
+            const routes = this.#routes.get(key) ?? [];
+            routes.push({
+                ownerKey,
+                descriptor: runtime.descriptor,
+                tenant: runtime.tenant,
+                tenantKey: runtime.tenant.tenantId === undefined
+                    ? undefined
+                    : String(TenantBoundary.from(runtime.tenant.tenantId).key),
+            });
+            this.#routes.set(key, routes);
+        }
+    }
+    get empty() {
+        return this.#routes.size === 0;
+    }
+    fenceOwners(ownerKeys) {
+        for (const ownerKey of ownerKeys)
+            this.#retiringOwners.add(ownerKey);
+    }
+    async awaitOwnersSettled(ownerKeys) {
+        await Promise.all(Array.from(this.#reserved.values(), async (reservations) => Promise.all(Array.from(reservations.values())
+            .filter((reservation) => ownerKeys.includes(reservation.route.ownerKey))
+            .map((reservation) => reservation.settled.promise))));
+        await Promise.all(ownerKeys.map(async (ownerKey) => {
+            const active = this.#active.get(ownerKey);
+            if (active !== undefined)
+                await Promise.allSettled(active);
+        }));
+    }
+    async retireOwners(ownerKeys) {
+        this.fenceOwners(ownerKeys);
+        await this.awaitOwnersSettled(ownerKeys);
+        for (const ownerKey of ownerKeys)
+            this.remove(ownerKey);
+        if (this.empty)
+            await this.retire();
+    }
+    remove(ownerKey) {
+        for (const [key, routes] of this.#routes) {
+            const retained = routes.filter((route) => route.ownerKey !== ownerKey);
+            if (retained.length === 0)
+                this.#routes.delete(key);
+            else
+                this.#routes.set(key, retained);
+        }
+    }
+    start() {
+        if (this.#stopped) {
+            return Promise.reject(new Error("Environment delivery supervisor is stopped."));
+        }
+        if (this.#start === undefined) {
+            this.#start = this.#supervisor.start().then(() => {
+                if (this.#source instanceof LocalDeliverySource)
+                    this.#source.enableRecovery();
+            });
+        }
+        return this.#start;
+    }
+    notify(shard) {
+        if (this.#stopped)
+            return;
+        // spine-log-boundary: server.delivery_notify_start
+        void this.start()
+            .then(() => {
+            this.#supervisor.notify(shard);
+        })
+            .catch(() => undefined);
+    }
+    stop() {
+        if (this.#stopped)
+            return;
+        this.#stopped = true;
+        this.#close = this.#supervisor.close();
+        // spine-log-boundary: server.delivery_supervisor_close
+        void this.#close.catch(() => undefined);
+    }
+    async awaitSettled() {
+        await this.#start;
+        await this.#close;
+    }
+    async retire() {
+        this.stop();
+        await this.awaitSettled();
+    }
+    #accept(message) {
+        const route = this.#select(message);
+        if (route !== undefined) {
+            // `options.shards` initializes one private reservation map per supervised shard.
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            const reservations = this.#reserved.get(message.shard.key());
+            reservations.set(_a.messageKey(message), {
+                route,
+                settled: Promise.withResolvers(),
+            });
+        }
+        return route !== undefined;
+    }
+    #route(message) {
+        const key = _a.messageKey(message);
+        // `Delivery` invokes the callback only after this group's admission reserved its route.
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const reservations = this.#reserved.get(message.shard.key());
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const reservation = reservations.get(key);
+        const route = reservation.route;
+        const active = this.#active.get(route.ownerKey) ?? new Set();
+        this.#active.set(route.ownerKey, active);
+        const replay = Promise.resolve()
+            .then(() => route.descriptor.replay(message, route.tenant.tenantId))
+            .finally(() => {
+            active.delete(replay);
+            if (active.size === 0)
+                this.#active.delete(route.ownerKey);
+        });
+        active.add(replay);
+        reservations.delete(key);
+        reservation.settled.resolve(undefined);
+        return replay;
+    }
+    #releaseUndeliveredReservations(shard) {
+        // `options.shards` initializes one private reservation map per supervised shard.
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const reservations = this.#reserved.get(shard.key());
+        for (const reservation of reservations.values()) {
+            reservation.settled.resolve(undefined);
+        }
+        reservations.clear();
+    }
+    #select(message) {
+        const key = _a.key(message);
+        const routes = this.#routes.get(key);
+        const route = _a.select(routes, message);
+        return route === undefined || this.#retiringOwners.has(route.ownerKey) ? undefined : route;
+    }
+    static select(routes, message) {
+        if (routes === undefined || routes.length === 0)
+            return undefined;
+        const tenantKey = _a.tenantKey(message);
+        return routes.find((route) => route.tenantKey === tenantKey);
+    }
+    /**
+     *
+     * Derives a tenant boundary key from the delivery label's matching envelope.
+     * `HANDLE_COMMAND` reads the Command actor tenant; Event labels read imported
+     * or past-message Event origin tenants.
+     *
+     * @param message Contains the admitted delivery message.
+     * @returns The tenant boundary key when the supported envelope carries one.
+     */
+    static tenantKey(message) {
+        if (message.signal === undefined)
+            return undefined;
+        try {
+            if (message.label === "HANDLE_COMMAND") {
+                const tenantId = fromBinary(CommandSchema, message.signal.value).context?.actorContext
+                    ?.tenantId;
+                return tenantId === undefined ? undefined : String(TenantBoundary.from(tenantId).key);
+            }
+            const origin = fromBinary(EventSchema, message.signal.value).context?.origin;
+            if (origin?.case === "importContext") {
+                const value = origin.value.tenantId;
+                return value === undefined ? undefined : String(TenantBoundary.from(value).key);
+            }
+            if (origin?.case === "pastMessage") {
+                const value = origin.value.actorContext?.tenantId;
+                return value === undefined ? undefined : String(TenantBoundary.from(value).key);
+            }
+        }
+        catch {
+            return undefined;
+        }
+        return undefined;
+    }
+    static key(route) {
+        const targetTypeUrl = route.inboxId?.targetTypeUrl ?? route.targetTypeUrl;
+        if (targetTypeUrl === undefined)
+            throw new Error("Environment delivery route has no target type.");
+        return JSON.stringify([route.label, targetTypeUrl, route.shard.index, route.shard.ofTotal]);
+    }
+    static messageKey(message) {
+        return `${message.signalId}/${_a.key(message)}`;
+    }
+}
+_a = RuntimeDeliverySupervisorGroup;
+class LocalDeliverySource {
+    #shards = new Map();
+    #recoveryEnabled = false;
+    constructor(shards) {
+        this.add(shards);
+    }
+    add(shards) {
+        for (const shard of shards)
+            this.#shards.set(shard.key(), shard);
+    }
+    enableRecovery() {
+        this.#recoveryEnabled = true;
+    }
+    shardSnapshot() {
+        return Promise.resolve(this.#recoveryEnabled
+            ? Array.from(this.#shards.values(), (shard) => ({
+                shard,
+                status: "NOT_PICKED",
+                messages: 1,
+            }))
+            : []);
+    }
+    observeShardUpdates(options) {
+        return EnvironmentDeliveryValues.updatesUntilAborted(options?.signal);
+    }
+    releaseExpired() {
+        return Promise.resolve([]);
+    }
+}
+/**
+ *
+ * @internal Groups private delivery-runtime assembly and failure operations.
+ */
+const EnvironmentDeliveryValues = Object.freeze({
+    requireSource(source) {
+        if (source === null ||
+            typeof source !== "object" ||
+            typeof source.shardSnapshot !== "function" ||
+            typeof source.observeShardUpdates !== "function" ||
+            typeof source.releaseExpired !== "function") {
+            throw new TypeError("Environment delivery source is invalid.");
+        }
+        return source;
+    },
+    createWorker(runtime, ports, nodeId) {
+        const delivery = new Delivery({
+            context: runtime.context,
+            storageFactory: runtime.storageFactory,
+            node: nodeId ?? runtime.context.name,
+            ...(ports ?? {}),
+        });
+        const worker = new DeliveryWorker({
+            delivery,
+            shards: EnvironmentDeliveryValues.uniqueShards(runtime.scopes),
+            onMessage: (message) => runtime.descriptor.replay(message, runtime.tenant.tenantId),
+        });
+        return deliveryRunWorkers.worker(worker);
+    },
+    createSupervisorGroup(runtime, shards, ports, nodeId) {
+        const first = shards[0];
+        if (first === undefined) {
+            throw new Error("Environment delivery supervisor requires at least one shard.");
+        }
+        const builder = new DeliveryBuilder()
+            .withContext(runtime.context)
+            .withStorageFactory(runtime.storageFactory)
+            .withStrategy(UniformAcrossAllShards.forNumber(first.ofTotal))
+            .withNode(nodeId ?? runtime.context.name);
+        if (ports !== undefined)
+            builder.withInbox(ports.inbox).withWorkRegistry(ports.workRegistry);
+        return new RuntimeDeliverySupervisorGroup({
+            delivery: builder.build(),
+            shards,
+            ...(ports?.source === undefined ? {} : { source: ports.source }),
+            ...(runtime.logger === undefined ? {} : { logger: runtime.logger }),
+        });
+    },
+    updatesUntilAborted(signal) {
+        return {
+            [Symbol.asyncIterator]() {
+                return {
+                    next: () => new Promise((resolve) => {
+                        if (signal?.aborted) {
+                            resolve({ done: true, value: undefined });
+                            return;
+                        }
+                        signal?.addEventListener("abort", () => {
+                            resolve({ done: true, value: undefined });
+                        }, { once: true });
+                    }),
+                };
+            },
+        };
+    },
+    soleOwner(scopes) {
+        const first = scopes[0]?.owner;
+        if (first === undefined || scopes.some(({ owner }) => owner.key !== first.key)) {
+            throw new Error("Environment delivery obligation requires exactly one owner.");
+        }
+        return first;
+    },
+    uniqueShards(scopes) {
+        const shards = new Map();
+        for (const { ready } of scopes) {
+            shards.set(ready.shard.key(), ready.shard);
+        }
+        return Object.freeze([...shards.values()]);
+    },
+    throwFailures(failures, message) {
+        if (failures.length === 1) {
+            throw failures[0];
+        }
+        if (failures.length > 1) {
+            throw new AggregateError(failures, message);
+        }
+    },
+    async attempt(action) {
+        await action();
+    },
+});
+//# sourceMappingURL=environment-delivery-worker.js.map