@oatscenter/trellis 0.100.0-rc.1

package/service/runtime/core.js

@@ -0,0 +1,1747 @@
+import { createInbox, headers, } from "@nats-io/nats-core";
+import { Pointer, Value } from "typebox/value";
+import { routeQueueGroup, } from "../../participant_runtime/api.js";
+import { AsyncResult, BaseError, err, isErr, ok, Result, } from "@oatscenter/result";
+import { ulid } from "ulid";
+import { encodeSchema, parseSchema } from "../../codec.js";
+import { canonicalizeJson, digestJson, } from "../../participant_runtime/json.js";
+import { AuthError, OperationAlreadyTerminalError, OperationNotFoundError, TransferError, UnexpectedError, ValidationError, } from "../../errors/index.js";
+import { serviceRuntimeLogger } from "./logger.js";
+import { recordTrellisError, } from "../../telemetry/mod.js";
+import { annotateHandlerBoundaryError, buildRuntimeOperationSnapshot, isOperationDeferred, isResultLike, isTerminalRuntimeOperationSnapshot, safeJson, Trellis, verifyLocalAuthorization, } from "../../session.js";
+import { FileInfoSchema, } from "../../transfer.js";
+const CANONICAL_POSITIVE_INTEGER = /^[1-9][0-9]*$/u;
+function asStringPointerValue(operation, input, pointer, field) {
+    const value = Pointer.Get(input, pointer);
+    if (typeof value !== "string" || value.length === 0) {
+        return err(new TransferError({
+            operation: "transfer",
+            context: { reason: "invalid_input", operation, field, pointer },
+        }));
+    }
+    return ok(value);
+}
+function asOptionalStringPointerValue(input, pointer) {
+    if (!pointer) {
+        return ok(undefined);
+    }
+    const value = Pointer.Get(input, pointer);
+    if (value === undefined) {
+        return ok(undefined);
+    }
+    if (typeof value !== "string" || value.length === 0) {
+        return err(new TransferError({
+            operation: "transfer",
+            context: { reason: "invalid_input", field: pointer, pointer },
+        }));
+    }
+    return ok(value);
+}
+function asOptionalStringRecordPointerValue(input, pointer) {
+    if (!pointer) {
+        return ok(undefined);
+    }
+    const value = Pointer.Get(input, pointer);
+    if (value === undefined) {
+        return ok(undefined);
+    }
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return err(new TransferError({
+            operation: "transfer",
+            context: { reason: "invalid_input", field: pointer, pointer },
+        }));
+    }
+    const entries = Object.entries(value);
+    if (entries.some(([key, item]) => key.length === 0 || typeof item !== "string")) {
+        return err(new TransferError({
+            operation: "transfer",
+            context: { reason: "invalid_input", field: pointer, pointer },
+        }));
+    }
+    return ok(Object.fromEntries(entries));
+}
+function traceIdFromTraceparent(traceparent) {
+    const [version, traceId, parentId, flags, extra] = traceparent?.split("-") ??
+        [];
+    if (extra !== undefined ||
+        !/^[0-9a-f]{2}$/u.test(version ?? "") ||
+        version === "ff" ||
+        !/^[0-9a-f]{32}$/u.test(traceId ?? "") ||
+        traceId === "00000000000000000000000000000000" ||
+        !/^[0-9a-f]{16}$/u.test(parentId ?? "") ||
+        parentId === "0000000000000000" ||
+        !/^[0-9a-f]{2}$/u.test(flags ?? "")) {
+        return undefined;
+    }
+    return traceId;
+}
+function recordOperationServiceError(error, attributes) {
+    recordTrellisError(error, {
+        messagingSystem: "nats",
+        surface: "operation",
+        direction: "server",
+        ...attributes,
+    });
+}
+function isOperationRevisionConflict(cause) {
+    const message = cause instanceof Error
+        ? `${cause.message} ${cause.cause instanceof Error ? cause.cause.message : ""}`
+            .toLowerCase()
+        : "";
+    return message.includes("operation revision conflict") ||
+        message.includes("wrong last sequence") ||
+        message.includes("wrong last revision") ||
+        message.includes("revision mismatch") ||
+        message.includes("sequence mismatch");
+}
+export class TrellisServiceRuntime extends Trellis {
+    #nats;
+    #version;
+    #log;
+    #operations = new Map();
+    #activeOperationFences = new Map();
+    #operationUpdateSequences = new Map();
+    #mountedOperationControls = new Set();
+    #stopPromise;
+    #transferSupport;
+    #operationOwnerId;
+    #operationConnectionId;
+    #operationDeploymentId;
+    operations;
+    constructor(name, nats, auth, opts) {
+        super(name, nats, auth, {
+            ...opts,
+            log: opts?.log ?? serviceRuntimeLogger,
+        });
+        if (opts?.operationDeploymentId) {
+            this.setOperationDeploymentId(opts.operationDeploymentId);
+        }
+        if (opts?.feedOwnerId)
+            this.setFeedOwnerId(opts.feedOwnerId);
+        this.#nats = nats;
+        this.#version = opts?.version;
+        this.#log = (opts?.log ?? serviceRuntimeLogger).child({
+            lib: "trellis-service-runtime",
+        });
+        this.#transferSupport = opts?.transferSupport;
+        this.#operationOwnerId = ulid();
+        this.#operationConnectionId = opts?.operationConnectionId ?? "";
+        this.#operationDeploymentId = opts?.operationDeploymentId;
+        this.operations = {
+            get: (operationId) => AsyncResult.from((async () => {
+                const runtime = await this.#resolveOperation(operationId);
+                if (!runtime) {
+                    return err(this.#operationNotFoundError(operationId));
+                }
+                return ok(runtime.snapshot);
+            })()),
+            started: (operationId) => this.#applyOwnedOperationUpdate(operationId, "running", {
+                event: { type: "started" },
+            }),
+            progress: (operationId, progress) => this.#applyOwnedOperationUpdate(operationId, "running", {
+                patch: { progress },
+                event: { type: "progress", progress },
+            }),
+            complete: (operationId, output) => this.#applyOwnedOperationUpdate(operationId, "completed", {
+                patch: { output },
+                event: { type: "completed" },
+            }),
+            fail: (operationId, error) => this.#applyOwnedOperationUpdate(operationId, "failed", {
+                patch: { error: error.toSerializable() },
+                event: { type: "failed" },
+            }),
+            cancel: (operationId) => this.#applyOwnedOperationUpdate(operationId, "cancelled", {
+                event: { type: "cancelled" },
+            }),
+            signals: (operationId) => this.#signals(operationId),
+            nextSignal: (operationId, name) => this.#nextSignal(operationId, name),
+        };
+    }
+    async *#signals(operationId) {
+        let cursor = 0;
+        while (true) {
+            const next = await this.#nextSignalAfter(operationId, cursor).take();
+            if (isErr(next)) {
+                throw next.error;
+            }
+            cursor = next.sequence;
+            yield next;
+        }
+    }
+    #nextSignal(operationId, name) {
+        return AsyncResult.from((async () => {
+            let cursor = 0;
+            while (true) {
+                const next = await this.#nextSignalAfter(operationId, cursor).take();
+                if (isErr(next))
+                    return next;
+                cursor = next.sequence;
+                if (!name || next.signal === name)
+                    return ok(next);
+            }
+        })());
+    }
+    #nextSignalAfter(operationId, afterSequence) {
+        return AsyncResult.from((async () => {
+            while (true) {
+                const runtime = await this.#resolveOperation(operationId);
+                if (!runtime) {
+                    return err(this.#operationNotFoundError(operationId));
+                }
+                const durable = await this.loadOperationRecord(operationId);
+                if (durable && durable.revision > runtime.revision) {
+                    runtime.revision = durable.revision;
+                    runtime.snapshot = durable.snapshot;
+                    runtime.sequence = durable.sequence;
+                    runtime.signalSequence = durable.signalSequence;
+                    runtime.signals = durable.signals;
+                    runtime.terminal = durable.snapshot.state === "completed" ||
+                        durable.snapshot.state === "failed" ||
+                        durable.snapshot.state === "cancelled";
+                    if (durable.cancelRequestedAt) {
+                        runtime.cancelRequestedAt = durable.cancelRequestedAt;
+                    }
+                }
+                const queued = runtime.signals.find((signal) => !signal.acknowledged && signal.sequence > afterSequence);
+                if (queued) {
+                    return ok(queued);
+                }
+                if (runtime.terminal) {
+                    return err(this.#operationAlreadyTerminalError(runtime));
+                }
+                await new Promise((resolve) => setTimeout(resolve, 50));
+            }
+        })());
+    }
+    async #resolveOperation(operationId) {
+        const existing = this.#operations.get(operationId);
+        const durable = await this.loadOperationRecord(operationId);
+        if (!durable)
+            return null;
+        if (existing?.ownerInstanceId === this.#operationOwnerId &&
+            durable.ownerInstanceId === this.#operationOwnerId &&
+            durable.ownerEpoch === existing.ownerEpoch) {
+            if (durable.revision > existing.revision) {
+                existing.revision = durable.revision;
+                existing.leaseExpiresAt = durable.leaseExpiresAt;
+                existing.cancelRequestedAt = durable.cancelRequestedAt;
+                existing.snapshot = durable.snapshot;
+                existing.sequence = durable.sequence;
+                existing.signalSequence = durable.signalSequence;
+                existing.signals = durable.signals;
+                existing.terminal = durable.snapshot.state === "completed" ||
+                    durable.snapshot.state === "failed" ||
+                    durable.snapshot.state === "cancelled";
+            }
+            if (existing.cancelRequestedAt || existing.terminal) {
+                existing.cancellation.abort("operation cancelled");
+            }
+            return existing;
+        }
+        if (existing) {
+            if (existing.ownerInstanceId === this.#operationOwnerId) {
+                existing.cancellation.abort("operation superseded by another owner");
+            }
+            this.#operations.delete(operationId);
+        }
+        const cancellation = new AbortController();
+        const runtime = {
+            id: durable.invocationId,
+            service: durable.snapshot.service,
+            operation: durable.snapshot.operation,
+            callerSessionKey: durable.callerSessionKey,
+            invocationDigest: durable.invocationDigest,
+            caller: durable.caller,
+            creatorPrincipalId: durable.creatorPrincipalId,
+            creatorParticipantId: durable.creatorParticipantId,
+            apiId: durable.apiId,
+            input: durable.input,
+            revision: durable.revision,
+            ownerInstanceId: durable.ownerInstanceId,
+            ownerConnectionId: durable.ownerConnectionId,
+            ownerEpoch: durable.ownerEpoch,
+            leaseExpiresAt: durable.leaseExpiresAt,
+            ...(durable.cancelRequestedAt
+                ? { cancelRequestedAt: durable.cancelRequestedAt }
+                : {}),
+            ...(durable.transferGrant
+                ? { transferGrant: durable.transferGrant }
+                : {}),
+            snapshot: durable.snapshot,
+            sequence: durable.sequence,
+            terminal: durable.snapshot.state === "completed" ||
+                durable.snapshot.state === "failed" ||
+                durable.snapshot.state === "cancelled",
+            signalSequence: durable.signalSequence,
+            signals: durable.signals,
+            watchers: new Map(),
+            frameQueue: Promise.resolve(),
+            signalWaiters: new Set(),
+            cancellation,
+        };
+        if (runtime.cancelRequestedAt || runtime.terminal) {
+            cancellation.abort("operation cancelled");
+        }
+        this.#operations.set(operationId, runtime);
+        return runtime;
+    }
+    async #acquireOperation(durable) {
+        if (durable.snapshot.state === "completed" ||
+            durable.snapshot.state === "failed" ||
+            durable.snapshot.state === "cancelled" ||
+            Date.parse(durable.leaseExpiresAt) > Date.now())
+            return null;
+        const existing = await this.#resolveOperation(durable.invocationId);
+        if (!existing || existing.revision !== durable.revision)
+            return null;
+        // Supersede any prior local execution before taking a new epoch so an
+        // earlier execution's continuations cannot receive this execution's
+        // cancellation or borrow its owner fence.
+        if (existing.ownerInstanceId === this.#operationOwnerId) {
+            existing.cancellation.abort("operation superseded by new acquisition");
+            this.#operations.delete(existing.id);
+        }
+        const acquired = await this.#resolveOperation(durable.invocationId);
+        if (!acquired || acquired.revision !== durable.revision)
+            return null;
+        const runtime = acquired;
+        runtime.ownerInstanceId = this.#operationOwnerId;
+        runtime.ownerConnectionId = this.#operationConnectionId;
+        runtime.ownerEpoch = durable.ownerEpoch + 1;
+        runtime.leaseExpiresAt = new Date(Date.now() + 30_000).toISOString();
+        try {
+            await this.saveOperationRecord(runtime);
+        }
+        catch {
+            if (this.#operations.get(runtime.id) === runtime) {
+                this.#operations.delete(runtime.id);
+            }
+            return null;
+        }
+        runtime.reclaimed = true;
+        this.#activeOperationFences.set(runtime.id, this.#operationFence(runtime));
+        return runtime;
+    }
+    #operationFence(runtime) {
+        return {
+            ownerInstanceId: runtime.ownerInstanceId,
+            ownerConnectionId: runtime.ownerConnectionId,
+            ownerEpoch: runtime.ownerEpoch,
+        };
+    }
+    #ownsOperation(durable, fence) {
+        return durable.ownerInstanceId === fence.ownerInstanceId &&
+            durable.ownerConnectionId === fence.ownerConnectionId &&
+            durable.ownerEpoch === fence.ownerEpoch &&
+            Date.parse(durable.leaseExpiresAt) > Date.now();
+    }
+    #releaseOperationFence(operationId, fence) {
+        const active = this.#activeOperationFences.get(operationId);
+        if (active?.ownerInstanceId === fence.ownerInstanceId &&
+            active.ownerConnectionId === fence.ownerConnectionId &&
+            active.ownerEpoch === fence.ownerEpoch) {
+            this.#activeOperationFences.delete(operationId);
+        }
+    }
+    #applyOperationUpdate(runtime, fence, state, opts) {
+        return AsyncResult.from((async () => {
+            return await this.#queueOperationFrame(runtime, async () => {
+                for (let retry = 0;; retry++) {
+                    const durable = await this.loadOperationRecord(runtime.id);
+                    if (!durable || !this.#ownsOperation(durable, fence)) {
+                        runtime.cancellation.abort("operation ownership lost");
+                        return err(this.#operationAlreadyTerminalError(runtime));
+                    }
+                    if (durable.snapshot.state === "completed" ||
+                        durable.snapshot.state === "failed" ||
+                        durable.snapshot.state === "cancelled" ||
+                        durable.cancelRequestedAt ||
+                        runtime.cancellation.signal.aborted)
+                        return err(this.#operationAlreadyTerminalError(runtime));
+                    runtime.revision = durable.revision;
+                    runtime.snapshot = durable.snapshot;
+                    runtime.sequence = durable.sequence + 1;
+                    runtime.signalSequence = durable.signalSequence;
+                    runtime.signals = durable.signals;
+                    runtime.leaseExpiresAt = new Date(Date.now() + 30_000).toISOString();
+                    runtime.snapshot = buildRuntimeOperationSnapshot(runtime, state, opts.patch);
+                    runtime.terminal = state === "completed" || state === "failed" ||
+                        state === "cancelled";
+                    try {
+                        await this.saveOperationRecord(runtime);
+                        break;
+                    }
+                    catch (cause) {
+                        if (retry >= 3 || !isOperationRevisionConflict(cause)) {
+                            runtime.cancellation.abort("operation ownership lost");
+                            throw cause;
+                        }
+                    }
+                }
+                if (runtime.terminal) {
+                    this.#releaseOperationFence(runtime.id, fence);
+                    this.#rejectSignalWaiters(runtime);
+                }
+                return ok(runtime.snapshot);
+            });
+        })());
+    }
+    #applyOwnedOperationUpdate(operationId, state, opts) {
+        return AsyncResult.from((async () => {
+            const runtime = await this.#resolveOperation(operationId);
+            if (!runtime)
+                return err(this.#operationNotFoundError(operationId));
+            const fence = this.#activeOperationFences.get(operationId);
+            if (!fence)
+                return err(this.#operationNotFoundError(operationId));
+            return await this.#applyOperationUpdate(runtime, fence, state, opts);
+        })());
+    }
+    async #queueOperationFrame(runtime, task) {
+        const previous = runtime.frameQueue;
+        let release;
+        runtime.frameQueue = new Promise((resolve) => {
+            release = resolve;
+        });
+        await previous;
+        try {
+            return await task();
+        }
+        finally {
+            release?.();
+        }
+    }
+    #emitOperationUpdate(runtime, fence, ctx, update) {
+        return this.#queueOperationFrame(runtime, async () => {
+            const durable = await this.loadOperationRecord(runtime.id);
+            if (runtime.terminal || runtime.cancellation.signal.aborted || !durable ||
+                durable.cancelRequestedAt || durable.snapshot.state === "cancelled" ||
+                !this.#ownsOperation(durable, fence)) {
+                runtime.cancellation.abort("operation cancelled or ownership lost");
+                return err(this.#operationAlreadyTerminalError(runtime));
+            }
+            if (ctx.update === undefined) {
+                return err(new ValidationError({
+                    errors: [{
+                            path: "/",
+                            message: `Operation '${runtime.operation}' does not declare updates`,
+                        }],
+                    context: { operation: runtime.operation },
+                }));
+            }
+            const parsed = this.#encodeOperationValue(ctx, "update", update).take();
+            if (isErr(parsed))
+                return parsed;
+            const key = `${runtime.id}:${fence.ownerEpoch}`;
+            const sequence = (this.#operationUpdateSequences.get(key) ?? 0) + 1;
+            this.#operationUpdateSequences.set(key, sequence);
+            const metadata = headers();
+            const occurredAt = new Date().toISOString();
+            const subject = `${ctx.subject}.updates.${runtime.id}`;
+            const payload = JSON.stringify({
+                operationId: runtime.id,
+                apiId: runtime.apiId,
+                operation: runtime.operation,
+                deploymentId: this.#operationDeploymentId,
+                ownerExecutorId: fence.ownerInstanceId,
+                ownerConnectionId: runtime.ownerConnectionId,
+                ownerEpoch: String(fence.ownerEpoch),
+                sequence: String(sequence),
+                occurredAt,
+                update: parsed,
+            });
+            const contextDigest = typeof this.auth.contextDigest === "function"
+                ? this.auth.contextDigest()
+                : this.auth.contextDigest;
+            const cache = this.auth.authorizationProviderCache;
+            if (!contextDigest || !cache) {
+                return err(new AuthError({ reason: "authorization_unavailable" }));
+            }
+            const ownContext = await cache.resolveContext(contextDigest);
+            const reply = createInbox(ownContext.context.inboxPrefix);
+            const authorization = await this.createRequestProof(subject, payload, reply);
+            metadata.set("proof", authorization.proof);
+            metadata.set("iat", String(authorization.iat));
+            metadata.set("request-id", authorization.requestId);
+            metadata.set("authorization-context", authorization.contextDigest);
+            metadata.set("session-key", this.auth.sessionKey);
+            this.#log.info({
+                operationId: runtime.id,
+                operation: runtime.operation,
+                ownerExecutorId: fence.ownerInstanceId,
+                ownerConnectionId: runtime.ownerConnectionId,
+                ownerEpoch: fence.ownerEpoch,
+                sequence,
+                subject,
+                reply,
+            }, "Publishing transient operation update");
+            await this.#nats.publish(subject, payload, { headers: metadata, reply });
+            await this.#nats.flush();
+            this.#log.info({
+                operationId: runtime.id,
+                ownerEpoch: fence.ownerEpoch,
+                sequence,
+            }, "Published transient operation update");
+            return ok(undefined);
+        });
+    }
+    #encodeOperationValue(ctx, kind, value) {
+        const schema = kind === "progress"
+            ? ctx.progress
+            : kind === "update"
+                ? ctx.update
+                : ctx.output;
+        if (schema === undefined)
+            return ok(value);
+        try {
+            if (schema && typeof schema === "object" &&
+                typeof Reflect.get(schema, "encode") === "function") {
+                return ok(Reflect.get(schema, "encode").call(schema, value));
+            }
+            const encoded = encodeSchema(schema, value).take();
+            return isErr(encoded) ? encoded : ok(JSON.parse(encoded));
+        }
+        catch (cause) {
+            return err(new UnexpectedError({ cause }));
+        }
+    }
+    #decodeOperationValue(ctx, kind, value) {
+        const schema = kind === "progress"
+            ? ctx.progress
+            : kind === "update"
+                ? ctx.update
+                : ctx.output;
+        if (schema === undefined)
+            return ok(value);
+        try {
+            if (schema && typeof schema === "object" &&
+                typeof Reflect.get(schema, "decode") === "function") {
+                return ok(Reflect.get(schema, "decode").call(schema, value));
+            }
+            return parseSchema(schema, value);
+        }
+        catch (cause) {
+            return err(new UnexpectedError({ cause }));
+        }
+    }
+    #applyControlledOperationUpdate(runtime, fence, ctx, state, opts) {
+        return AsyncResult.from((async () => {
+            if (runtime.ownerInstanceId !== fence.ownerInstanceId ||
+                runtime.ownerEpoch !== fence.ownerEpoch) {
+                return err(new UnexpectedError({
+                    cause: new Error("operation update rejected by owner fence"),
+                }));
+            }
+            runtime.leaseExpiresAt = new Date(Date.now() + 30_000).toISOString();
+            if (opts.patch?.progress !== undefined) {
+                const parsed = this.#encodeOperationValue(ctx, "progress", opts.patch.progress).take();
+                if (isErr(parsed))
+                    return parsed;
+                opts.patch.progress = parsed;
+                if ("progress" in opts.event)
+                    opts.event.progress = parsed;
+            }
+            if (opts.patch?.output !== undefined) {
+                const parsed = this.#encodeOperationValue(ctx, "output", opts.patch.output).take();
+                if (isErr(parsed))
+                    return parsed;
+                opts.patch.output = parsed;
+            }
+            return await this.#applyOperationUpdate(runtime, fence, state, opts);
+        })());
+    }
+    #controlOperation(operation, ctx, operationId) {
+        return AsyncResult.from((async () => {
+            const runtime = await this.#resolveOperation(operationId);
+            if (!runtime) {
+                return err(this.#operationNotFoundError(operationId));
+            }
+            if (!this.#matchesOperationRoute(runtime, operation, ctx)) {
+                return err(this.#operationNotFoundError(operationId));
+            }
+            const fence = this.#activeOperationFences.get(operationId);
+            if (!fence)
+                return err(this.#operationNotFoundError(operationId));
+            return ok(this.#makeControlledOperation(runtime, fence, ctx));
+        })());
+    }
+    #makeControlledOperation(runtime, fence, ctx) {
+        return {
+            id: runtime.id,
+            started: () => this.#applyControlledOperationUpdate(runtime, fence, ctx, "running", {
+                event: { type: "started" },
+            }),
+            progress: (value) => this.#applyControlledOperationUpdate(runtime, fence, ctx, "running", {
+                patch: { progress: value },
+                event: { type: "progress", progress: value },
+            }),
+            emitUpdate: (value) => AsyncResult.from(this.#emitOperationUpdate(runtime, fence, ctx, value)),
+            complete: (value) => this.#applyControlledOperationUpdate(runtime, fence, ctx, "completed", {
+                patch: { output: value },
+                event: { type: "completed" },
+            }),
+            fail: (error) => this.#applyControlledOperationUpdate(runtime, fence, ctx, "failed", {
+                patch: { error: error.toSerializable() },
+                event: { type: "failed" },
+            }),
+            cancel: () => this.#applyControlledOperationUpdate(runtime, fence, ctx, "cancelled", {
+                event: { type: "cancelled" },
+            }),
+            attach: (job) => AsyncResult.from((async () => {
+                const waited = await job.wait();
+                const waitedValue = waited.take();
+                if (isErr(waitedValue)) {
+                    return err(new UnexpectedError({ cause: waitedValue.error }));
+                }
+                const finalRuntime = await this.#resolveOperation(runtime.id);
+                if (!finalRuntime || !finalRuntime.terminal) {
+                    return err(new UnexpectedError({
+                        cause: new Error("attached job completed without terminal operation state"),
+                    }));
+                }
+                return ok(finalRuntime.snapshot);
+            })()),
+            signals: () => this.#signals(runtime.id),
+            nextSignal: (name) => this.#nextSignal(runtime.id, name),
+            acknowledgeSignal: (sequence) => this.#acknowledgeSignal(runtime, fence, sequence),
+            defer: () => ({ kind: "deferred" }),
+        };
+    }
+    #operationNotFoundError(operationId) {
+        return new OperationNotFoundError({ operationId });
+    }
+    #operationAlreadyTerminalError(runtime) {
+        return new OperationAlreadyTerminalError({
+            operationId: runtime.id,
+            state: runtime.snapshot.state,
+            operation: runtime.operation,
+            service: runtime.service,
+        });
+    }
+    #matchesOperationRoute(runtime, operation, ctx, caller) {
+        const invocationId = "invocationId" in runtime
+            ? runtime.invocationId
+            : runtime.id;
+        const apiId = `${ctx.permissions?.invoke.apiId ?? ""}@${ctx.permissions?.invoke.apiVersion ?? ""}`;
+        return runtime.apiId === apiId && runtime.operation === operation &&
+            runtime.snapshot.id === invocationId &&
+            runtime.snapshot.operation === operation &&
+            (!caller ||
+                (runtime.creatorPrincipalId === caller.principalId &&
+                    runtime.creatorParticipantId === caller.participantId));
+    }
+    #rejectSignalWaiters(runtime) {
+        const result = err(this.#operationAlreadyTerminalError(runtime));
+        for (const waiter of runtime.signalWaiters) {
+            waiter(result);
+        }
+        runtime.signalWaiters.clear();
+    }
+    async #acceptSignal(runtime, ctx, control, requestId, retry = 0) {
+        if (runtime.terminal) {
+            return err(this.#operationAlreadyTerminalError(runtime));
+        }
+        const replay = runtime.signals.find((signal) => signal.requestId === requestId);
+        if (replay) {
+            if (replay.signal !== control.signal ||
+                canonicalizeJson((replay.input ?? null)) !==
+                    canonicalizeJson((control.input ?? null))) {
+                return err(new ValidationError({
+                    errors: [{
+                            path: "/requestId",
+                            message: "Signal request id was already accepted with different input",
+                        }],
+                }));
+            }
+            return ok({
+                kind: "signal-accepted",
+                operationId: runtime.id,
+                signal: replay.signal,
+                signalSequence: replay.sequence,
+                acceptedAt: replay.acceptedAt,
+                snapshot: runtime.snapshot,
+            });
+        }
+        if (runtime.signals.length >= 100) {
+            return err(new ValidationError({
+                errors: [{
+                        path: "/signal",
+                        message: "Operation signal limit exceeded",
+                    }],
+            }));
+        }
+        const descriptor = ctx.signals?.[control.signal];
+        if (!descriptor) {
+            return err(new ValidationError({
+                errors: [{
+                        path: "/signal",
+                        message: `Unknown operation signal '${control.signal}'`,
+                    }],
+                context: { operation: runtime.operation, signal: control.signal },
+            }));
+        }
+        const input = control.input;
+        const parsed = parseSchema(descriptor.input, input).take();
+        if (isErr(parsed)) {
+            return err(parsed.error);
+        }
+        const encodedInput = new TextEncoder().encode(JSON.stringify(control.input ?? null)).byteLength;
+        if (encodedInput > 64 * 1024) {
+            return err(new ValidationError({
+                errors: [{
+                        path: "/input",
+                        message: "Operation signal payload exceeds 64 KiB",
+                    }],
+            }));
+        }
+        runtime.signalSequence += 1;
+        const acceptedAt = new Date().toISOString();
+        const signal = {
+            operationId: runtime.id,
+            sequence: runtime.signalSequence,
+            requestId,
+            signal: control.signal,
+            ...(control.input !== undefined ? { input: control.input } : {}),
+            acceptedAt,
+            acknowledged: false,
+        };
+        runtime.signals.push(signal);
+        try {
+            await this.saveOperationRecord(runtime);
+        }
+        catch (cause) {
+            this.#operations.delete(runtime.id);
+            if (retry < 3 && isOperationRevisionConflict(cause)) {
+                const current = await this.#resolveOperation(runtime.id);
+                if (current) {
+                    return await this.#acceptSignal(current, ctx, control, requestId, retry + 1);
+                }
+            }
+            throw cause;
+        }
+        const result = ok(signal);
+        for (const waiter of runtime.signalWaiters) {
+            waiter(result);
+        }
+        return ok({
+            kind: "signal-accepted",
+            operationId: runtime.id,
+            signal: signal.signal,
+            signalSequence: signal.sequence,
+            acceptedAt,
+            snapshot: runtime.snapshot,
+        });
+    }
+    #acknowledgeSignal(runtime, fence, sequence) {
+        return AsyncResult.from((async () => {
+            return await this.#queueOperationFrame(runtime, async () => {
+                for (let retry = 0;; retry++) {
+                    const durable = await this.loadOperationRecord(runtime.id);
+                    if (!durable || !this.#ownsOperation(durable, fence) ||
+                        durable.snapshot.state === "completed" ||
+                        durable.snapshot.state === "failed" ||
+                        durable.snapshot.state === "cancelled" || durable.cancelRequestedAt) {
+                        runtime.cancellation.abort("operation ownership lost");
+                        return err(this.#operationAlreadyTerminalError(runtime));
+                    }
+                    runtime.revision = durable.revision;
+                    runtime.snapshot = durable.snapshot;
+                    runtime.sequence = durable.sequence;
+                    runtime.signalSequence = durable.signalSequence;
+                    runtime.signals = durable.signals;
+                    const signal = runtime.signals.find((item) => item.sequence === sequence);
+                    if (!signal) {
+                        return err(new ValidationError({
+                            errors: [{
+                                    path: "/sequence",
+                                    message: `Operation signal ${sequence} was not found`,
+                                }],
+                        }));
+                    }
+                    if (signal.acknowledged)
+                        return ok(undefined);
+                    signal.acknowledged = true;
+                    try {
+                        await this.saveOperationRecord(runtime);
+                        return ok(undefined);
+                    }
+                    catch (cause) {
+                        if (retry >= 3 || !isOperationRevisionConflict(cause)) {
+                            runtime.cancellation.abort("operation ownership lost");
+                            throw cause;
+                        }
+                    }
+                }
+            });
+        })());
+    }
+    async #authenticateOperationMessage(msg, ctx, parseInput, permission = ctx.permissions?.invoke, requiredCapabilities = ctx.callerCapabilities ?? []) {
+        const jsonData = safeJson(msg).take();
+        if (isErr(jsonData))
+            return jsonData;
+        let parsedInput;
+        let invocationId;
+        if (parseInput) {
+            if (!jsonData || typeof jsonData !== "object" || Array.isArray(jsonData) ||
+                typeof jsonData.invocationId !==
+                    "string" ||
+                !/^[0-9A-HJKMNP-TV-Z]{26}$/u.test(jsonData.invocationId) || !("input" in jsonData)) {
+                return err(new ValidationError({
+                    errors: [{
+                            path: "",
+                            message: "Operation start requires a ULID invocationId and input",
+                        }],
+                }));
+            }
+            invocationId = jsonData.invocationId;
+            const parsedInputResult = parseSchema(ctx.input, jsonData.input).take();
+            if (isErr(parsedInputResult)) {
+                return err(parsedInputResult.error);
+            }
+            parsedInput = parsedInputResult;
+        }
+        else {
+            parsedInput = jsonData;
+        }
+        const auth = await verifyLocalAuthorization({
+            kind: "request",
+            cache: this.auth.authorizationProviderCache,
+            message: msg,
+            permission,
+            requiredCapabilities,
+        });
+        const authValue = auth.take();
+        if (isErr(authValue))
+            return err(authValue.error);
+        return ok({
+            input: parsedInput,
+            ...(invocationId ? { invocationId } : {}),
+            caller: authValue,
+            sessionKey: authValue.sessionKey,
+        });
+    }
+    #ensureOperationControlLoop(operation, ctx) {
+        const controlSubject = `${ctx.subject}.control`;
+        if (this.#mountedOperationControls.has(controlSubject)) {
+            return;
+        }
+        this.#mountedOperationControls.add(controlSubject);
+        const publishFrame = async (reply, frame) => {
+            await this.#nats.publish(reply, JSON.stringify(frame));
+        };
+        const publishSnapshot = async (reply, snapshot) => {
+            await publishFrame(reply, { kind: "snapshot", snapshot });
+        };
+        const respondControlError = (msg, error) => {
+            const trellisError = error instanceof BaseError
+                ? error
+                : new UnexpectedError({ cause: error });
+            recordOperationServiceError(trellisError, {
+                operation,
+                phase: "control",
+            });
+            msg.respond(JSON.stringify({
+                kind: "error",
+                error: trellisError.toSerializable(),
+            }));
+        };
+        const controlSub = this.#nats.subscribe(controlSubject, {
+            queue: routeQueueGroup(controlSubject),
+        });
+        void (async () => {
+            for await (const msg of controlSub) {
+                const request = safeJson(msg).take();
+                if (isErr(request)) {
+                    respondControlError(msg, request.error);
+                    continue;
+                }
+                if (!request ||
+                    typeof request !== "object" ||
+                    !["get", "watch", "signal", "cancel"].includes(String(request.action)) ||
+                    typeof request.operationId !==
+                        "string" ||
+                    (request.action === "signal" &&
+                        typeof request.signal !== "string")) {
+                    respondControlError(msg, new UnexpectedError({
+                        cause: new Error("Invalid operation control request"),
+                    }));
+                    continue;
+                }
+                const control = request;
+                if (control.action === "signal" && !ctx.signals?.[control.signal]) {
+                    respondControlError(msg, new ValidationError({
+                        errors: [{
+                                path: "/signal",
+                                message: `Unknown operation signal '${control.signal}'`,
+                            }],
+                    }));
+                    continue;
+                }
+                let permission = ctx.permissions?.observe;
+                let capabilities = ctx.observeCapabilities ?? [];
+                if (control.action === "cancel") {
+                    permission = ctx.permissions?.cancel;
+                    capabilities = ctx.cancelCapabilities ?? [];
+                }
+                else if (control.action === "signal") {
+                    permission = ctx.permissions?.control[control.signal];
+                    capabilities = ctx.controlCapabilities ?? [];
+                }
+                const validated = await this.#authenticateOperationMessage(msg, ctx, false, permission, capabilities);
+                const value = validated.take();
+                if (isErr(value)) {
+                    respondControlError(msg, value.error);
+                    continue;
+                }
+                let runtime;
+                try {
+                    runtime = await this.#resolveOperation(control.operationId);
+                }
+                catch (cause) {
+                    respondControlError(msg, cause instanceof Error ? cause : new Error(String(cause)));
+                    continue;
+                }
+                if (!runtime) {
+                    respondControlError(msg, this.#operationNotFoundError(control.operationId));
+                    continue;
+                }
+                if (!this.#matchesOperationRoute(runtime, operation, ctx, value.caller)) {
+                    respondControlError(msg, this.#operationNotFoundError(control.operationId));
+                    continue;
+                }
+                const snapshot = runtime.snapshot;
+                if (control.action === "watch") {
+                    if (msg.reply) {
+                        const reply = msg.reply;
+                        const updateSub = control.includeUpdates
+                            ? this.#nats.subscribe(`${ctx.subject}.updates.${control.operationId}`)
+                            : undefined;
+                        void (async () => {
+                            let ownerEpoch = 0;
+                            let updateSequence = 0;
+                            let revision = runtime.revision;
+                            const durableWatch = (await (await this.operationStoreHandle())
+                                .watch(control.operationId).orThrow())[Symbol.asyncIterator]();
+                            if (updateSub)
+                                await this.#nats.flush();
+                            await publishSnapshot(reply, runtime.snapshot);
+                            const updates = updateSub
+                                ? (async () => {
+                                    for await (const updateMsg of updateSub) {
+                                        try {
+                                            const cache = this.auth.authorizationProviderCache;
+                                            if (!cache)
+                                                throw new Error("authorization unavailable");
+                                            await cache.resolveContext(value.caller.contextDigest);
+                                        }
+                                        catch {
+                                            updateSub.unsubscribe();
+                                            await durableWatch.return?.();
+                                            break;
+                                        }
+                                        const providerAuthorization = await verifyLocalAuthorization({
+                                            kind: "request",
+                                            cache: this.auth.authorizationProviderCache,
+                                            message: updateMsg,
+                                            permission: undefined,
+                                            requiredCapabilities: [],
+                                            identityOnly: true,
+                                        });
+                                        const provider = providerAuthorization.take();
+                                        if (isErr(provider))
+                                            continue;
+                                        const decoded = safeJson(updateMsg).take();
+                                        if (isErr(decoded) || !decoded ||
+                                            typeof decoded !== "object" || Array.isArray(decoded))
+                                            continue;
+                                        const envelope = decoded;
+                                        const current = await this.loadOperationRecord(control.operationId);
+                                        const executor = envelope.ownerExecutorId;
+                                        const connectionId = envelope.ownerConnectionId;
+                                        const epoch = typeof envelope.ownerEpoch === "string" &&
+                                            CANONICAL_POSITIVE_INTEGER.test(envelope.ownerEpoch)
+                                            ? Number(envelope.ownerEpoch)
+                                            : NaN;
+                                        const sequence = typeof envelope.sequence === "string" &&
+                                            CANONICAL_POSITIVE_INTEGER.test(envelope.sequence)
+                                            ? Number(envelope.sequence)
+                                            : NaN;
+                                        const parsedUpdate = this.#decodeOperationValue(ctx, "update", envelope.update).take();
+                                        if (isErr(parsedUpdate) ||
+                                            typeof executor !== "string" ||
+                                            typeof connectionId !== "string" ||
+                                            typeof envelope.occurredAt !== "string" ||
+                                            envelope.operationId !== control.operationId ||
+                                            envelope.apiId !== current?.apiId ||
+                                            envelope.operation !== current?.operation ||
+                                            envelope.deploymentId !== this.#operationDeploymentId ||
+                                            !current ||
+                                            !this.#matchesOperationRoute(current, operation, ctx, value.caller) || current.snapshot.state === "completed" ||
+                                            current.snapshot.state === "failed" ||
+                                            current.snapshot.state === "cancelled" ||
+                                            current.cancelRequestedAt ||
+                                            Date.parse(current.leaseExpiresAt) <= Date.now() ||
+                                            current.ownerEpoch !== epoch ||
+                                            current.ownerInstanceId !== executor ||
+                                            current.ownerConnectionId !== connectionId ||
+                                            provider.participantId !== this.contractId ||
+                                            provider.deploymentId !== this.#operationDeploymentId ||
+                                            provider.connectionId !== current.ownerConnectionId ||
+                                            !Number.isSafeInteger(epoch) ||
+                                            !Number.isSafeInteger(sequence) ||
+                                            epoch < ownerEpoch ||
+                                            (epoch === ownerEpoch && sequence <= updateSequence))
+                                            continue;
+                                        ownerEpoch = epoch;
+                                        updateSequence = sequence;
+                                        const wireUpdate = this.#encodeOperationValue(ctx, "update", parsedUpdate).take();
+                                        if (isErr(wireUpdate))
+                                            continue;
+                                        await publishFrame(reply, {
+                                            kind: "event",
+                                            sequence: current.sequence,
+                                            event: {
+                                                type: "update",
+                                                update: wireUpdate,
+                                                snapshot: current.snapshot,
+                                            },
+                                        });
+                                    }
+                                })()
+                                : Promise.resolve();
+                            try {
+                                for (;;) {
+                                    const next = await durableWatch.next();
+                                    if (next.done)
+                                        break;
+                                    const changed = next.value;
+                                    const durable = changed.take();
+                                    if (isErr(durable))
+                                        throw durable.error;
+                                    if (!durable.value)
+                                        break;
+                                    if (durable.value.revision <= revision)
+                                        continue;
+                                    revision = durable.value.revision;
+                                    if (!this.#matchesOperationRoute(durable.value, operation, ctx, value.caller))
+                                        break;
+                                    try {
+                                        const cache = this.auth.authorizationProviderCache;
+                                        if (!cache)
+                                            throw new Error("authorization unavailable");
+                                        await cache.resolveContext(value.caller.contextDigest);
+                                    }
+                                    catch {
+                                        break;
+                                    }
+                                    await publishSnapshot(reply, durable.value.snapshot);
+                                    if (durable.value.snapshot.state === "completed" ||
+                                        durable.value.snapshot.state === "failed" ||
+                                        durable.value.snapshot.state === "cancelled")
+                                        break;
+                                }
+                            }
+                            finally {
+                                updateSub?.unsubscribe();
+                                await updates;
+                            }
+                        })().catch((error) => {
+                            if (!this.#nats.isClosed()) {
+                                this.#log.warn({ error, operation: String(operation) }, "Operation watch stopped");
+                            }
+                        });
+                    }
+                    continue;
+                }
+                if (control.action === "get") {
+                    msg.respond(JSON.stringify({ kind: "snapshot", snapshot }));
+                    continue;
+                }
+                if (control.action === "cancel") {
+                    if (runtime.terminal) {
+                        respondControlError(msg, this.#operationAlreadyTerminalError(runtime));
+                        continue;
+                    }
+                    try {
+                        let current = runtime;
+                        for (let retry = 0;; retry++) {
+                            current.cancelRequestedAt = new Date().toISOString();
+                            current.sequence += 1;
+                            current.snapshot = buildRuntimeOperationSnapshot(current, "cancelled", { completedAt: current.cancelRequestedAt });
+                            current.terminal = true;
+                            try {
+                                await this.saveOperationRecord(current);
+                                break;
+                            }
+                            catch (cause) {
+                                this.#operations.delete(current.id);
+                                if (retry >= 3 || !isOperationRevisionConflict(cause)) {
+                                    throw cause;
+                                }
+                                const reloaded = await this.#resolveOperation(current.id);
+                                if (!reloaded || reloaded.terminal)
+                                    throw cause;
+                                current = reloaded;
+                            }
+                        }
+                        current.cancellation.abort("operation cancelled");
+                        msg.respond(JSON.stringify({ kind: "snapshot", snapshot: current.snapshot }));
+                    }
+                    catch (cause) {
+                        respondControlError(msg, new UnexpectedError({ cause }));
+                    }
+                    continue;
+                }
+                if (control.action === "signal") {
+                    if (!runtime) {
+                        respondControlError(msg, new UnexpectedError({
+                            cause: new Error("operation is not running in this process"),
+                        }));
+                        continue;
+                    }
+                    try {
+                        const accepted = await this.#acceptSignal(runtime, ctx, control, msg.headers?.get("request-id") ?? ulid());
+                        const acceptedValue = accepted.take();
+                        if (isErr(acceptedValue)) {
+                            respondControlError(msg, acceptedValue.error);
+                            continue;
+                        }
+                        msg.respond(JSON.stringify(acceptedValue));
+                    }
+                    catch (cause) {
+                        respondControlError(msg, cause instanceof Error ? cause : new Error(String(cause)));
+                        continue;
+                    }
+                    continue;
+                }
+                respondControlError(msg, new UnexpectedError({
+                    cause: new Error(`Unknown operation control action '${control.action}' for '${operation}'`),
+                }));
+            }
+        })();
+    }
+    mountRuntime(method, fn) {
+        return super.mount(method, fn);
+    }
+    static create(name, nats, auth, opts) {
+        const runtime = new TrellisServiceRuntime(name, nats, auth, opts);
+        return runtime;
+    }
+    operationHandle(operation) {
+        const ctx = this.api["operations"]?.[operation];
+        if (!ctx) {
+            throw new Error(`Unknown operation '${operation.toString()}'. Did you forget to include its API module?`);
+        }
+        return {
+            control: (operationId) => {
+                this.#ensureOperationControlLoop(String(operation), ctx);
+                return this.#controlOperation(String(operation), ctx, operationId);
+            },
+            handle: async (handler) => {
+                const startSubject = ctx.subject;
+                const now = () => new Date().toISOString();
+                const publishFrame = async (reply, frame) => {
+                    await this.#nats.publish(reply, JSON.stringify(frame));
+                };
+                const makeOperation = (runtime, fence, context) => {
+                    return {
+                        id: runtime.id,
+                        started: () => this.#applyControlledOperationUpdate(runtime, fence, ctx, "running", {
+                            event: { type: "started" },
+                        }),
+                        progress: (value) => this.#applyControlledOperationUpdate(runtime, fence, ctx, "running", {
+                            patch: { progress: value },
+                            event: { type: "progress", progress: value },
+                        }),
+                        emitUpdate: (value) => AsyncResult.from(this.#emitOperationUpdate(runtime, fence, ctx, value)),
+                        complete: (value) => this.#applyControlledOperationUpdate(runtime, fence, ctx, "completed", {
+                            patch: { output: value, completedAt: now() },
+                            event: { type: "completed" },
+                        }),
+                        fail: (error) => AsyncResult.from((async () => {
+                            const annotatedError = annotateHandlerBoundaryError(error, {
+                                operation: String(operation),
+                                requestId: context.requestId,
+                                service: this.name,
+                                contractId: this.contractId,
+                                contractDigest: this.contractDigest,
+                                traceId: context.traceId,
+                            });
+                            return await this.#applyControlledOperationUpdate(runtime, fence, ctx, "failed", {
+                                patch: {
+                                    error: annotatedError.toSerializable(),
+                                    completedAt: now(),
+                                },
+                                event: { type: "failed" },
+                            });
+                        })()),
+                        cancel: () => this.#applyControlledOperationUpdate(runtime, fence, ctx, "cancelled", {
+                            patch: { completedAt: now() },
+                            event: { type: "cancelled" },
+                        }),
+                        attach: (job) => AsyncResult.from((async () => {
+                            const waited = await job.wait();
+                            const waitedValue = waited.take();
+                            if (isErr(waitedValue)) {
+                                return err(new UnexpectedError({ cause: waitedValue.error }));
+                            }
+                            const finalRuntime = await this.loadOperationRecord(runtime.id);
+                            if (!finalRuntime || !this.#ownsOperation(finalRuntime, fence) ||
+                                !isTerminalRuntimeOperationSnapshot(finalRuntime.snapshot)) {
+                                return err(new UnexpectedError({
+                                    cause: new Error("attached job completed without terminal operation state"),
+                                }));
+                            }
+                            return ok(finalRuntime.snapshot);
+                        })()),
+                        signals: () => this.#signals(runtime.id),
+                        nextSignal: (name) => this.#nextSignal(runtime.id, name),
+                        acknowledgeSignal: (sequence) => this.#acknowledgeSignal(runtime, fence, sequence),
+                        defer: () => ({ kind: "deferred" }),
+                    };
+                };
+                const executeHandler = async (runtime, fence, caller, transferSession, operationContext = {}, resuming = false) => {
+                    const admitted = await this.loadOperationRecord(runtime.id);
+                    if (!admitted || !this.#ownsOperation(admitted, fence) ||
+                        admitted.snapshot.state === "completed" ||
+                        admitted.snapshot.state === "failed" ||
+                        admitted.snapshot.state === "cancelled" ||
+                        admitted.cancelRequestedAt ||
+                        !this.#matchesOperationRoute(admitted, String(operation), ctx, caller)) {
+                        runtime.cancellation.abort("operation ownership lost");
+                        this.#releaseOperationFence(runtime.id, fence);
+                        return;
+                    }
+                    runtime.revision = admitted.revision;
+                    runtime.leaseExpiresAt = admitted.leaseExpiresAt;
+                    runtime.snapshot = admitted.snapshot;
+                    const op = makeOperation(runtime, fence, operationContext);
+                    try {
+                        const handlerResult = await handler(transferSession
+                            ? {
+                                input: runtime.input,
+                                op,
+                                caller,
+                                signal: runtime.cancellation.signal,
+                                resuming,
+                                ...(runtime.snapshot.progress !== undefined
+                                    ? { progress: runtime.snapshot.progress }
+                                    : {}),
+                                transfer: transferSession.transfer,
+                            }
+                            : {
+                                input: runtime.input,
+                                op,
+                                caller,
+                                signal: runtime.cancellation.signal,
+                                resuming,
+                                ...(runtime.snapshot.progress !== undefined
+                                    ? { progress: runtime.snapshot.progress }
+                                    : {}),
+                            });
+                        const handlerOutcome = isResultLike(handlerResult)
+                            ? handlerResult.take()
+                            : handlerResult;
+                        if (isErr(handlerOutcome)) {
+                            const error = annotateHandlerBoundaryError(handlerOutcome.error, {
+                                operation: String(operation),
+                                requestId: operationContext.requestId,
+                                service: this.name,
+                                contractId: this.contractId,
+                                contractDigest: this.contractDigest,
+                                traceId: operationContext.traceId,
+                            });
+                            recordOperationServiceError(error, {
+                                operation: String(operation),
+                                phase: "handler_result",
+                            });
+                            await op.fail(error);
+                            return;
+                        }
+                        if (isOperationDeferred(handlerOutcome))
+                            return;
+                        if (isTerminalRuntimeOperationSnapshot(handlerOutcome)) {
+                            return;
+                        }
+                        if (!runtime.terminal)
+                            await op.complete(handlerOutcome);
+                    }
+                    catch (cause) {
+                        if (runtime.cancellation.signal.aborted)
+                            return;
+                        const error = annotateHandlerBoundaryError(cause, {
+                            operation: String(operation),
+                            requestId: operationContext.requestId,
+                            service: this.name,
+                            contractId: this.contractId,
+                            contractDigest: this.contractDigest,
+                            traceId: operationContext.traceId,
+                        });
+                        recordOperationServiceError(error, {
+                            operation: String(operation),
+                            phase: "handler_throw",
+                        });
+                        try {
+                            await op.fail(error).take();
+                        }
+                        catch (failure) {
+                            recordOperationServiceError(failure, {
+                                operation: String(operation),
+                                phase: "failure_persist",
+                            });
+                        }
+                    }
+                };
+                const startLeaseHeartbeat = (runtime, fence) => {
+                    const leaseHeartbeat = setInterval(() => {
+                        if (runtime.terminal || runtime.cancellation.signal.aborted) {
+                            clearInterval(leaseHeartbeat);
+                            return;
+                        }
+                        void this.#queueOperationFrame(runtime, async () => {
+                            for (let retry = 0;; retry++) {
+                                const durable = await this.loadOperationRecord(runtime.id);
+                                if (!durable || durable.cancelRequestedAt ||
+                                    durable.snapshot.state === "cancelled" ||
+                                    !this.#ownsOperation(durable, fence))
+                                    throw new Error("operation ownership lost");
+                                runtime.revision = durable.revision;
+                                runtime.snapshot = durable.snapshot;
+                                runtime.sequence = durable.sequence;
+                                runtime.signalSequence = durable.signalSequence;
+                                runtime.signals = durable.signals;
+                                runtime.leaseExpiresAt = new Date(Date.now() + 30_000)
+                                    .toISOString();
+                                try {
+                                    await this.saveOperationRecord(runtime);
+                                    return;
+                                }
+                                catch (cause) {
+                                    if (retry >= 3 || !isOperationRevisionConflict(cause)) {
+                                        throw cause;
+                                    }
+                                }
+                            }
+                        }).catch(() => {
+                            runtime.cancellation.abort("operation ownership lost");
+                            if (this.#operations.get(runtime.id) === runtime) {
+                                this.#operations.delete(runtime.id);
+                            }
+                            this.#releaseOperationFence(runtime.id, fence);
+                            clearInterval(leaseHeartbeat);
+                        });
+                    }, 10_000);
+                };
+                const watchCancellation = (runtime, fence) => {
+                    let reading = false;
+                    const cancellationWatch = setInterval(() => {
+                        if (reading || runtime.terminal || runtime.cancellation.signal.aborted) {
+                            if (runtime.terminal || runtime.cancellation.signal.aborted) {
+                                clearInterval(cancellationWatch);
+                            }
+                            return;
+                        }
+                        reading = true;
+                        void this.loadOperationRecord(runtime.id).then((durable) => {
+                            if (!durable || !this.#ownsOperation(durable, fence)) {
+                                runtime.cancellation.abort("operation cancelled or ownership lost");
+                                clearInterval(cancellationWatch);
+                            }
+                        }).catch(() => {
+                            runtime.cancellation.abort("operation state unavailable");
+                            clearInterval(cancellationWatch);
+                        }).finally(() => {
+                            reading = false;
+                        });
+                    }, 100);
+                };
+                const authenticate = (msg, parseInput = true) => this.#authenticateOperationMessage(msg, ctx, parseInput);
+                this.#log.info({ operation: String(operation) }, `Mounting ${String(operation)} operation handler`);
+                this.#ensureOperationControlLoop(String(operation), ctx);
+                const recover = async (durable) => {
+                    if (!this.#matchesOperationRoute(durable, String(operation), ctx)) {
+                        return;
+                    }
+                    const runtime = await this.#acquireOperation(durable);
+                    if (!runtime?.reclaimed)
+                        return;
+                    runtime.reclaimed = false;
+                    if (runtime.transferGrant) {
+                        const fence = this.#operationFence(runtime);
+                        const committed = Reflect.get(runtime.transferGrant, "committed");
+                        if (Value.Check(FileInfoSchema, committed)) {
+                            startLeaseHeartbeat(runtime, fence);
+                            watchCancellation(runtime, fence);
+                            void executeHandler(runtime, fence, runtime.caller, {
+                                grant: runtime.transferGrant,
+                                transfer: {
+                                    updates: async function* () { },
+                                    completed: () => AsyncResult.from(Promise.resolve(ok(committed))),
+                                },
+                            }, {}, true);
+                            return;
+                        }
+                        if (!ctx.transfer || !this.#transferSupport)
+                            return;
+                        const key = asStringPointerValue(String(operation), runtime.input, ctx.transfer.key, "key").take();
+                        const contentType = asOptionalStringPointerValue(runtime.input, ctx.transfer.contentType).take();
+                        const metadata = asOptionalStringRecordPointerValue(runtime.input, ctx.transfer.metadata).take();
+                        if (isErr(key) || isErr(contentType) || isErr(metadata))
+                            return;
+                        const reopened = await this.#transferSupport
+                            .openOperationTransfer({
+                            sessionKey: runtime.callerSessionKey,
+                            permission: ctx.permissions?.invoke,
+                            requiredCapabilities: ctx.callerCapabilities ?? [],
+                            store: ctx.transfer.store,
+                            key,
+                            expiresInMs: ctx.transfer.expiresInMs ?? 60_000,
+                            ...(ctx.transfer.maxBytes !== undefined
+                                ? { maxBytes: ctx.transfer.maxBytes }
+                                : {}),
+                            ...(contentType !== undefined ? { contentType } : {}),
+                            ...(metadata !== undefined ? { metadata } : {}),
+                            onComplete: async (info) => {
+                                await this.#queueOperationFrame(runtime, async () => {
+                                    if (!runtime.transferGrant)
+                                        return;
+                                    const durable = await this.loadOperationRecord(runtime.id);
+                                    if (!durable || !this.#ownsOperation(durable, fence) ||
+                                        durable.cancelRequestedAt ||
+                                        durable.snapshot.state === "completed" ||
+                                        durable.snapshot.state === "failed" ||
+                                        durable.snapshot.state === "cancelled") {
+                                        runtime.cancellation.abort("operation ownership lost");
+                                        return;
+                                    }
+                                    runtime.revision = durable.revision;
+                                    Reflect.set(runtime.transferGrant, "committed", info);
+                                    await this.saveOperationRecord(runtime);
+                                });
+                            },
+                        }).take();
+                        if (isErr(reopened))
+                            return;
+                        runtime.transferGrant = reopened.grant;
+                        const current = await this.loadOperationRecord(runtime.id);
+                        if (!current || !this.#ownsOperation(current, fence)) {
+                            runtime.cancellation.abort("operation ownership lost");
+                            return;
+                        }
+                        runtime.revision = current.revision;
+                        await this.saveOperationRecord(runtime);
+                        startLeaseHeartbeat(runtime, fence);
+                        void (async () => {
+                            for await (const progress of reopened.transfer.updates()) {
+                                await this.#applyOperationUpdate(runtime, fence, "running", {
+                                    patch: { transfer: progress },
+                                    event: { type: "transfer", transfer: progress },
+                                });
+                            }
+                        })();
+                        watchCancellation(runtime, fence);
+                        void executeHandler(runtime, fence, runtime.caller, reopened, {}, true);
+                        return;
+                    }
+                    const fence = this.#operationFence(runtime);
+                    startLeaseHeartbeat(runtime, fence);
+                    watchCancellation(runtime, fence);
+                    void executeHandler(runtime, fence, runtime.caller, undefined, {}, true);
+                };
+                const recoverExpired = async () => {
+                    for (const durable of await this.listNonterminalOperationRecords()) {
+                        if (Date.parse(durable.leaseExpiresAt) <= Date.now()) {
+                            await recover(durable);
+                        }
+                    }
+                };
+                await recoverExpired();
+                const recoveryScan = setInterval(() => {
+                    if (this.#nats.isClosed()) {
+                        clearInterval(recoveryScan);
+                        return;
+                    }
+                    void recoverExpired().catch((error) => {
+                        if (!this.#nats.isClosed()) {
+                            this.#log.warn({ error, operation: String(operation) }, "Operation recovery scan failed");
+                        }
+                    });
+                }, 1_000);
+                const startSub = this.#nats.subscribe(startSubject, {
+                    queue: routeQueueGroup(startSubject),
+                });
+                void (async () => {
+                    for await (const msg of startSub) {
+                        const validated = await authenticate(msg, true);
+                        const value = validated.take();
+                        if (isErr(value)) {
+                            recordOperationServiceError(value.error, {
+                                operation: String(operation),
+                                phase: "start",
+                            });
+                            this.respondWithError(msg, value.error);
+                            continue;
+                        }
+                        let transferSession;
+                        let transferFence;
+                        const operationId = value.invocationId;
+                        const apiId = `${ctx.permissions?.invoke.apiId ?? ""}@${ctx.permissions?.invoke.apiVersion ?? ""}`;
+                        const invocationDigest = (await digestJson({
+                            apiId,
+                            operation: String(operation),
+                            creatorPrincipalId: value.caller.principalId,
+                            creatorParticipantId: value.caller.participantId,
+                            input: value.input,
+                        })).digest;
+                        let reclaimed;
+                        const existing = await this.#resolveOperation(operationId);
+                        if (existing) {
+                            if (!this.#matchesOperationRoute(existing, String(operation), ctx, value.caller)) {
+                                this.respondWithError(msg, this.#operationNotFoundError(operationId));
+                                continue;
+                            }
+                            if (existing.invocationDigest !== invocationDigest) {
+                                this.respondWithError(msg, new ValidationError({
+                                    errors: [{
+                                            path: "/invocationId",
+                                            message: "Invocation id was already accepted with different input",
+                                        }],
+                                }));
+                                continue;
+                            }
+                            if (existing.reclaimed && !ctx.transfer) {
+                                reclaimed = existing;
+                            }
+                            else {
+                                msg.respond(JSON.stringify({
+                                    kind: "accepted",
+                                    ref: {
+                                        id: existing.id,
+                                        service: this.name,
+                                        operation: String(operation),
+                                    },
+                                    snapshot: existing.snapshot,
+                                    ...(existing.transferGrant
+                                        ? { transfer: existing.transferGrant }
+                                        : {}),
+                                }));
+                                continue;
+                            }
+                        }
+                        if (ctx.transfer) {
+                            if (!this.#transferSupport) {
+                                const error = new UnexpectedError({
+                                    cause: new Error(`Operation '${String(operation)}' declared transfer support but no runtime transfer support is configured`),
+                                });
+                                recordOperationServiceError(error, {
+                                    operation: String(operation),
+                                    phase: "start",
+                                });
+                                this.respondWithError(msg, error);
+                                continue;
+                            }
+                            const key = ctx.transfer.key
+                                ? asStringPointerValue(String(operation), value.input, ctx.transfer.key, "key").take()
+                                : operationId;
+                            if (isErr(key)) {
+                                recordOperationServiceError(key.error, {
+                                    operation: String(operation),
+                                    phase: "start",
+                                });
+                                this.respondWithError(msg, key.error);
+                                continue;
+                            }
+                            const contentType = asOptionalStringPointerValue(value.input, ctx.transfer.contentType).take();
+                            if (isErr(contentType)) {
+                                recordOperationServiceError(contentType.error, {
+                                    operation: String(operation),
+                                    phase: "start",
+                                });
+                                this.respondWithError(msg, contentType.error);
+                                continue;
+                            }
+                            const metadata = asOptionalStringRecordPointerValue(value.input, ctx.transfer.metadata).take();
+                            if (isErr(metadata)) {
+                                recordOperationServiceError(metadata.error, {
+                                    operation: String(operation),
+                                    phase: "start",
+                                });
+                                this.respondWithError(msg, metadata.error);
+                                continue;
+                            }
+                            const openedTransferValue = await this.#transferSupport
+                                .openOperationTransfer({
+                                sessionKey: value.sessionKey,
+                                permission: ctx.permissions?.invoke,
+                                requiredCapabilities: ctx.callerCapabilities ?? [],
+                                store: ctx.transfer.store,
+                                key,
+                                expiresInMs: ctx.transfer.expiresInMs ?? 60_000,
+                                ...(ctx.transfer.maxBytes !== undefined
+                                    ? { maxBytes: ctx.transfer.maxBytes }
+                                    : {}),
+                                ...(contentType !== undefined ? { contentType } : {}),
+                                ...(metadata !== undefined ? { metadata } : {}),
+                                onComplete: async (info) => {
+                                    await this.#queueOperationFrame(runtime, async () => {
+                                        if (!runtime.transferGrant || !transferFence)
+                                            return;
+                                        const durable = await this.loadOperationRecord(runtime.id);
+                                        if (!durable ||
+                                            !this.#ownsOperation(durable, transferFence) ||
+                                            durable.cancelRequestedAt ||
+                                            durable.snapshot.state === "completed" ||
+                                            durable.snapshot.state === "failed" ||
+                                            durable.snapshot.state === "cancelled") {
+                                            runtime.cancellation.abort("operation ownership lost");
+                                            return;
+                                        }
+                                        runtime.revision = durable.revision;
+                                        Reflect.set(runtime.transferGrant, "committed", info);
+                                        await this.saveOperationRecord(runtime);
+                                    });
+                                },
+                            }).take();
+                            if (isErr(openedTransferValue)) {
+                                recordOperationServiceError(openedTransferValue.error, {
+                                    operation: String(operation),
+                                    phase: "start",
+                                });
+                                this.respondWithError(msg, openedTransferValue.error);
+                                continue;
+                            }
+                            transferSession = openedTransferValue;
+                        }
+                        const createdAt = now();
+                        const runtime = reclaimed ?? {
+                            id: operationId,
+                            service: this.name,
+                            operation: String(operation),
+                            callerSessionKey: value.sessionKey,
+                            invocationDigest,
+                            caller: value.caller,
+                            creatorPrincipalId: value.caller.principalId,
+                            creatorParticipantId: value.caller.participantId,
+                            apiId,
+                            input: value.input,
+                            revision: 1,
+                            ownerInstanceId: this.#operationOwnerId,
+                            ownerConnectionId: this.#operationConnectionId,
+                            ownerEpoch: 1,
+                            leaseExpiresAt: new Date(Date.now() + 30_000).toISOString(),
+                            ...(transferSession
+                                ? { transferGrant: transferSession.grant }
+                                : {}),
+                            snapshot: {
+                                id: operationId,
+                                service: this.name,
+                                operation: String(operation),
+                                revision: 1,
+                                state: "pending",
+                                createdAt,
+                                updatedAt: createdAt,
+                            },
+                            sequence: 0,
+                            signalSequence: 0,
+                            signals: [],
+                            terminal: false,
+                            watchers: new Map(),
+                            frameQueue: Promise.resolve(),
+                            signalWaiters: new Set(),
+                            cancellation: new AbortController(),
+                        };
+                        transferFence = this.#operationFence(runtime);
+                        this.#activeOperationFences.set(runtime.id, transferFence);
+                        if (!reclaimed) {
+                            this.#operations.set(operationId, runtime);
+                            try {
+                                await this.saveOperationRecord(runtime);
+                            }
+                            catch (cause) {
+                                this.#operations.delete(operationId);
+                                const accepted = await this.#resolveOperation(operationId);
+                                if (!accepted) {
+                                    this.respondWithError(msg, cause instanceof Error
+                                        ? new ValidationError({
+                                            errors: [{ path: "/", message: cause.message }],
+                                        })
+                                        : new UnexpectedError({ cause }));
+                                    continue;
+                                }
+                                if (!this.#matchesOperationRoute(accepted, String(operation), ctx, value.caller) ||
+                                    accepted.invocationDigest !== invocationDigest) {
+                                    this.respondWithError(msg, new ValidationError({
+                                        errors: [{
+                                                path: "/invocationId",
+                                                message: "Invocation id was already accepted with different input",
+                                            }],
+                                    }));
+                                    continue;
+                                }
+                                msg.respond(JSON.stringify({
+                                    kind: "accepted",
+                                    ref: {
+                                        id: accepted.id,
+                                        service: this.name,
+                                        operation: String(operation),
+                                    },
+                                    snapshot: accepted.snapshot,
+                                    ...(accepted.transferGrant
+                                        ? { transfer: accepted.transferGrant }
+                                        : {}),
+                                }));
+                                continue;
+                            }
+                        }
+                        if (transferSession) {
+                            void (async () => {
+                                for await (const progress of transferSession.transfer.updates()) {
+                                    await this.#applyOperationUpdate(runtime, transferFence, "running", {
+                                        patch: { transfer: progress },
+                                        event: { type: "transfer", transfer: progress },
+                                    });
+                                }
+                            })();
+                        }
+                        startLeaseHeartbeat(runtime, transferFence);
+                        watchCancellation(runtime, transferFence);
+                        const accepted = {
+                            kind: "accepted",
+                            ref: {
+                                id: operationId,
+                                service: this.name,
+                                operation: String(operation),
+                            },
+                            snapshot: runtime.snapshot,
+                            ...(transferSession ? { transfer: transferSession.grant } : {}),
+                        };
+                        msg.respond(JSON.stringify(accepted));
+                        void executeHandler(runtime, transferFence, value.caller, transferSession, {
+                            requestId: msg.headers?.get("request-id"),
+                            traceId: traceIdFromTraceparent(msg.headers?.get("traceparent")),
+                        });
+                    }
+                })();
+                return Promise.resolve();
+            },
+        };
+    }
+    async stop() {
+        this.#stopPromise ??= (async () => {
+            if (this.#nats.isClosed()) {
+                return;
+            }
+            try {
+                await this.#nats.drain();
+            }
+            catch (cause) {
+                if (!(cause instanceof Error) ||
+                    cause.name !== "DrainingConnectionError") {
+                    throw cause;
+                }
+                await this.#nats.closed().catch(() => undefined);
+            }
+        })();
+        await this.#stopPromise;
+    }
+}