@qlever-llc/trellis 0.7.0 → 0.8.0-rc.3

package/esm/trellis.js

@@ -9,23 +9,29 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
     if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
     return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
 };
-var _Trellis_instances, _Trellis_log, _Trellis_tasks, _Trellis_hasExplicitApi, _Trellis_noResponderMaxRetries, _Trellis_noResponderRetryMs, _Trellis_authBypassMethods, _Trellis_operationStore, _Trellis_unknownApiError, _Trellis_handleRPC, _Trellis_processRPCMessage, _Trellis_respondWithError, _Trellis_startEphemeralEvent, _Trellis_handleDurableEvent, _Trellis_parseEventMessage, _Trellis_escapeSubjectToken, _Trellis_createProof, _Trellis_requestJson, _Trellis_watchJson;
+var _Trellis_instances, _Trellis_log, _Trellis_tasks, _Trellis_hasExplicitApi, _Trellis_noResponderMaxRetries, _Trellis_noResponderRetryMs, _Trellis_authBypassMethods, _Trellis_onSessionNotFound, _Trellis_operationStore, _Trellis_createStateFacade, _Trellis_unknownApiError, _Trellis_requestBuiltRpcUnknown, _Trellis_requestBuiltRpc, _Trellis_handleSessionNotFound, _Trellis_handleRPC, _Trellis_processRPCMessage, _Trellis_respondWithError, _Trellis_startEphemeralEvent, _Trellis_handleDurableEvent, _Trellis_parseEventMessage, _Trellis_escapeSubjectToken, _Trellis_createProof, _Trellis_requestMessageWithRetry, _Trellis_requestJson, _Trellis_watchJson;
 import { jetstream, jetstreamManager, } from "@nats-io/jetstream";
 import { createInbox, headers as natsHeaders, } from "@nats-io/nats-core";
+import { CONTRACT_JOBS_METADATA, CONTRACT_KV_METADATA, CONTRACT_STATE_METADATA, } from "./contract_support/mod.js";
 import { AsyncResult, BaseError, err, isErr, ok, Result, } from "@qlever-llc/result";
 import { createNatsHeaderCarrier, extractTraceContext, injectTraceContext, SpanStatusCode, startClientSpan, startServerSpan, withSpanAsync, } from "./tracing.js";
 import { Type } from "typebox";
 import { Pointer } from "typebox/value";
 import { ulid } from "ulid";
 import { encodeSchema, parse, parseUnknownSchema, } from "./codec.js";
-import { TransferError, AuthError, getBuiltinRpcError, UnexpectedError, ValidationError, } from "./errors/index.js";
+import { AuthError, getBuiltinRpcError, TransferError, TransportError, UnexpectedError, ValidationError, } from "./errors/index.js";
 import { RemoteError } from "./errors/RemoteError.js";
 import { logger } from "./globals.js";
 import { TypedKV } from "./kv.js";
 import { TrellisErrorDataSchema } from "./models/trellis/TrellisError.js";
 import { OperationInvoker, } from "./operations.js";
+import { StateDeleteResponseSchema, StateDeleteSchema, } from "./models/trellis/rpc/StateDelete.js";
+import { StateGetResponseSchema, StateGetSchema, } from "./models/trellis/rpc/StateGet.js";
+import { StateListResponseSchema, StateListSchema, } from "./models/trellis/rpc/StateList.js";
+import { StatePutResponseSchema, StatePutSchema, } from "./models/trellis/rpc/StatePut.js";
 import { createTransferHandle, } from "./transfer.js";
 import { TrellisTasks } from "./tasks.js";
+import { TrellisConnection } from "./connection.js";
 /**
  * Safely extract JSON from a NATS message.
  * The .json() method can throw if the message data is not valid JSON.
@@ -33,6 +39,74 @@ import { TrellisTasks } from "./tasks.js";
 export function safeJson(msg) {
     return Result.try(() => msg.json());
 }
+function transportCauseContext(cause) {
+    if (cause instanceof Error) {
+        return {
+            causeName: cause.name,
+            causeMessage: cause.message,
+        };
+    }
+    return { cause: String(cause) };
+}
+function createTransportError(args) {
+    return new TransportError({
+        code: args.code,
+        message: args.message,
+        hint: args.hint,
+        cause: args.cause,
+        context: {
+            ...(args.context ?? {}),
+            ...(args.cause === undefined ? {} : transportCauseContext(args.cause)),
+        },
+    });
+}
+function requestFailedTransportError(args) {
+    return createTransportError({
+        code: args.code,
+        message: args.message,
+        hint: args.hint,
+        cause: args.cause,
+        context: {
+            subject: args.subject,
+            ...(args.method === undefined ? {} : { method: args.method }),
+            ...(args.context ?? {}),
+        },
+    });
+}
+function classifyRequestTransportFailure(args) {
+    const message = args.cause instanceof Error
+        ? args.cause.message
+        : String(args.cause);
+    const isNoResponders = message.includes("no responders");
+    const isNatsPermission = message.includes("Permissions Violation");
+    return requestFailedTransportError({
+        code: isNoResponders
+            ? "trellis.request.unavailable"
+            : isNatsPermission
+                ? "trellis.request.denied"
+                : "trellis.request.failed",
+        message: isNoResponders
+            ? "Trellis could not reach the requested capability."
+            : isNatsPermission
+                ? "Trellis denied this request."
+                : "Trellis could not complete the request.",
+        hint: isNoResponders
+            ? "Check that the target service is installed and reachable, then try again."
+            : isNatsPermission
+                ? "Sign in with a profile that has the required capability, then try again."
+                : "Retry the request. If it keeps failing, check Trellis runtime health.",
+        cause: args.cause,
+        method: args.method,
+        subject: args.subject,
+        context: {
+            ...(args.callerCapabilities === undefined
+                ? {}
+                : { requiredCapabilities: args.callerCapabilities }),
+            noResponders: isNoResponders,
+            lowLevelMessage: message,
+        },
+    });
+}
 function encodeRuntimeSchema(schema, data) {
     return encodeSchema(schema, data);
 }
@@ -91,6 +165,14 @@ export function buildProofInput(sessionKey, subject, payloadHash) {
     buf.set(payloadHash, offset);
     return buf;
 }
+/**
+ * Returns true when a handler result explicitly leaves operation completion to
+ * an external control path.
+ */
+export function isOperationDeferred(value) {
+    return !!value && typeof value === "object" &&
+        "kind" in value && value.kind === "deferred";
+}
 export function isResultLike(value) {
     return value instanceof Result;
 }
@@ -176,6 +258,205 @@ export function isTerminalRuntimeOperationSnapshot(value) {
     return isRuntimeOperationSnapshot(value) && (value.state === "completed" || value.state === "failed" ||
         value.state === "cancelled");
 }
+const DEFAULT_STATE_LIST_LIMIT = 100;
+const STATE_RUNTIME_RPC = {
+    get: {
+        subject: "rpc.v1.State.Get",
+        input: StateGetSchema,
+        output: StateGetResponseSchema,
+        callerCapabilities: [],
+        errors: ["AuthError", "ValidationError", "UnexpectedError"],
+        declaredErrorTypes: [
+            "AuthError",
+            "ValidationError",
+            "UnexpectedError",
+        ],
+    },
+    put: {
+        subject: "rpc.v1.State.Put",
+        input: StatePutSchema,
+        output: StatePutResponseSchema,
+        callerCapabilities: [],
+        errors: ["AuthError", "ValidationError", "UnexpectedError"],
+        declaredErrorTypes: [
+            "AuthError",
+            "ValidationError",
+            "UnexpectedError",
+        ],
+    },
+    delete: {
+        subject: "rpc.v1.State.Delete",
+        input: StateDeleteSchema,
+        output: StateDeleteResponseSchema,
+        callerCapabilities: [],
+        errors: ["AuthError", "ValidationError", "UnexpectedError"],
+        declaredErrorTypes: [
+            "AuthError",
+            "ValidationError",
+            "UnexpectedError",
+        ],
+    },
+    list: {
+        subject: "rpc.v1.State.List",
+        input: StateListSchema,
+        output: StateListResponseSchema,
+        callerCapabilities: [],
+        errors: ["AuthError", "ValidationError", "UnexpectedError"],
+        declaredErrorTypes: [
+            "AuthError",
+            "ValidationError",
+            "UnexpectedError",
+        ],
+    },
+};
+function joinStatePath(prefix, key) {
+    return [prefix, key]
+        .flatMap((value) => value?.split("/") ?? [])
+        .filter((segment) => segment.length > 0)
+        .join("/");
+}
+function validateStateValue(schema, value) {
+    return parseRuntimeSchema(schema, value);
+}
+function validateStateGetResult(descriptor, result) {
+    if ("migrationRequired" in result) {
+        const schema = descriptor.acceptedVersions?.[result.stateVersion];
+        if (!schema) {
+            return Result.err(new ValidationError({
+                errors: [{
+                        path: "/stateVersion",
+                        message: `state version '${result.stateVersion}' is not accepted by the runtime store`,
+                    }],
+            }));
+        }
+        const parsed = validateStateValue(schema, result.entry.value);
+        if (parsed.isErr())
+            return Result.err(parsed.error);
+        return Result.ok({
+            ...result,
+            entry: {
+                ...result.entry,
+                value: parsed.unwrapOrElse(() => {
+                    throw new Error("state value validation unexpectedly failed");
+                }),
+            },
+        });
+    }
+    if (!result.found) {
+        return Result.ok(result);
+    }
+    const parsed = validateStateValue(descriptor.schema, result.entry.value);
+    if (parsed.isErr()) {
+        return Result.err(parsed.error);
+    }
+    return Result.ok({
+        ...result,
+        entry: {
+            ...result.entry,
+            value: parsed.unwrapOrElse(() => {
+                throw new Error("state value validation unexpectedly failed");
+            }),
+        },
+    });
+}
+function validateStatePutResult(descriptor, result) {
+    if (result.applied) {
+        const parsed = validateStateValue(descriptor.schema, result.entry.value);
+        if (parsed.isErr())
+            return Result.err(parsed.error);
+        return Result.ok({
+            ...result,
+            entry: {
+                ...result.entry,
+                value: parsed.unwrapOrElse(() => {
+                    throw new Error("state value validation unexpectedly failed");
+                }),
+            },
+        });
+    }
+    if (!result.entry) {
+        return Result.ok(result);
+    }
+    if ("migrationRequired" in result.entry) {
+        const schema = descriptor.acceptedVersions?.[result.entry.stateVersion];
+        if (!schema) {
+            return Result.err(new ValidationError({
+                errors: [{
+                        path: "/stateVersion",
+                        message: `state version '${result.entry.stateVersion}' is not accepted by the runtime store`,
+                    }],
+            }));
+        }
+        const parsed = validateStateValue(schema, result.entry.entry.value);
+        if (parsed.isErr())
+            return Result.err(parsed.error);
+        return Result.ok({
+            ...result,
+            entry: {
+                ...result.entry,
+                entry: {
+                    ...result.entry.entry,
+                    value: parsed.unwrapOrElse(() => {
+                        throw new Error("state value validation unexpectedly failed");
+                    }),
+                },
+            },
+        });
+    }
+    const parsed = validateStateValue(descriptor.schema, result.entry.value);
+    if (parsed.isErr()) {
+        return Result.err(parsed.error);
+    }
+    return Result.ok({
+        ...result,
+        entry: {
+            ...result.entry,
+            value: parsed.unwrapOrElse(() => {
+                throw new Error("state value validation unexpectedly failed");
+            }),
+        },
+    });
+}
+function validateStateListResult(descriptor, result) {
+    const entries = [];
+    for (const entry of result.entries) {
+        if ("migrationRequired" in entry) {
+            const schema = descriptor.acceptedVersions?.[entry.stateVersion];
+            if (!schema) {
+                return Result.err(new ValidationError({
+                    errors: [{
+                            path: "/stateVersion",
+                            message: `state version '${entry.stateVersion}' is not accepted by the runtime store`,
+                        }],
+                }));
+            }
+            const parsed = validateStateValue(schema, entry.entry.value);
+            if (parsed.isErr())
+                return Result.err(parsed.error);
+            entries.push({
+                ...entry,
+                entry: {
+                    ...entry.entry,
+                    value: parsed.unwrapOrElse(() => {
+                        throw new Error("state value validation unexpectedly failed");
+                    }),
+                },
+            });
+            continue;
+        }
+        const parsed = validateStateValue(descriptor.schema, entry.value);
+        if (parsed.isErr()) {
+            return Result.err(parsed.error);
+        }
+        entries.push({
+            ...entry,
+            value: parsed.unwrapOrElse(() => {
+                throw new Error("state value validation unexpectedly failed");
+            }),
+        });
+    }
+    return Result.ok({ ...result, entries });
+}
 const NATS_SUBJECT_TOKEN_FORBIDDEN = /[\u0000\s.*>~]/gu;
 const DEFAULT_NO_RESPONDER_MAX_RETRIES = 2;
 const DEFAULT_NO_RESPONDER_RETRY_MS = 200;
@@ -262,6 +543,19 @@ export class Trellis {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "state", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Framework-neutral lifecycle handle for this Trellis runtime connection. */
+        Object.defineProperty(this, "connection", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "nats", {
             enumerable: true,
             configurable: true,
@@ -292,6 +586,7 @@ export class Trellis {
         _Trellis_noResponderMaxRetries.set(this, void 0);
         _Trellis_noResponderRetryMs.set(this, void 0);
         _Trellis_authBypassMethods.set(this, void 0);
+        _Trellis_onSessionNotFound.set(this, void 0);
         _Trellis_operationStore.set(this, void 0);
         const api = opts?.api;
         this.name = name;
@@ -308,7 +603,11 @@ export class Trellis {
         __classPrivateFieldSet(this, _Trellis_noResponderRetryMs, opts?.noResponderRetry?.baseDelayMs ??
             DEFAULT_NO_RESPONDER_RETRY_MS, "f");
         __classPrivateFieldSet(this, _Trellis_authBypassMethods, new Set(opts?.authBypassMethods ?? []), "f");
+        __classPrivateFieldSet(this, _Trellis_onSessionNotFound, opts?.onSessionNotFound, "f");
+        this.connection = opts?.connection ??
+            new TrellisConnection({ kind: "client" });
         __classPrivateFieldSet(this, _Trellis_tasks, new TrellisTasks({ log: __classPrivateFieldGet(this, _Trellis_log, "f") }), "f");
+        this.state = __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_createStateFacade).call(this, opts?.state);
     }
     /**
      * Returns the underlying NATS connection.
@@ -351,161 +650,16 @@ export class Trellis {
         };
         await store.put(runtime.id, record);
     }
-    /**
-     * Makes an authenticated request to a Trellis RPC method.
-     *
-     * @template M The specific RPC method being called.
-     * @param method The name of the RPC method to call.
-     * @param input The input data for the method, conforming to its schema.
-     * @param opts Optional request-specific options.
-     * @returns A promise that resolves with a `Result` containing either the method's
-     * output or an error.
-     * @returns A `Result` object:
-     *              ok: A validated reponse of method M
-     *              err: declared RPC errors | RemoteError | ValidationError | UnexpectedError
-     */
-    async request(method, input, opts) {
-        __classPrivateFieldGet(this, _Trellis_log, "f").trace({ method: String(method) }, `Calling ${method.toString()}.`);
-        const methodName = method;
-        const ctx = this.api["rpc"][methodName];
+    request(method, input, opts) {
+        const rpcApi = this.api["rpc"];
+        const ctx = rpcApi[method];
         if (!ctx) {
-            return err(new UnexpectedError({
+            return AsyncResult.from(Promise.resolve(err(new UnexpectedError({
                 cause: __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_unknownApiError).call(this, "RPC method", method.toString()),
                 context: { method: method.toString() },
-            }));
-        }
-        const msg = encodeRuntimeSchema(ctx.input, input).take();
-        if (isErr(msg)) {
-            return msg;
-        }
-        const subject = this.template(ctx.subject, input).take();
-        if (isErr(subject)) {
-            return subject;
+            }))));
         }
-        // Start a client span for this RPC request
-        const span = startClientSpan(method, subject);
-        const attempt = async () => {
-            const proof = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_createProof).call(this, subject, msg);
-            const headers = natsHeaders();
-            headers.set("session-key", this.auth.sessionKey);
-            headers.set("proof", proof);
-            // Inject trace context into NATS headers for propagation
-            injectTraceContext(createNatsHeaderCarrier(headers), span);
-            // Attempt request with retry for transient "no responders" errors
-            const requestWithRetry = async () => {
-                for (let retry = 0; retry <= __classPrivateFieldGet(this, _Trellis_noResponderMaxRetries, "f"); retry++) {
-                    const result = await AsyncResult.try(() => this.nats.request(subject, msg, {
-                        headers,
-                        timeout: opts?.timeout ?? this.timeout,
-                    }));
-                    if (result.isOk()) {
-                        return ok((await result).take());
-                    }
-                    const cause = result.error.cause;
-                    const message = cause instanceof Error
-                        ? cause.message
-                        : String(cause);
-                    const isNoResponders = message.includes("no responders");
-                    // If it's a no-responders error and we have retries left, retry
-                    if (isNoResponders && retry < __classPrivateFieldGet(this, _Trellis_noResponderMaxRetries, "f")) {
-                        __classPrivateFieldGet(this, _Trellis_log, "f").debug({ method, subject, retry }, "No responders, retrying...");
-                        await new Promise((r) => setTimeout(r, __classPrivateFieldGet(this, _Trellis_noResponderRetryMs, "f") * (retry + 1)));
-                        continue;
-                    }
-                    // Final attempt failed or non-retryable error
-                    __classPrivateFieldGet(this, _Trellis_log, "f").warn({ method, subject, error: message }, "NATS request failed");
-                    const isNatsPermission = message.includes("Permissions Violation");
-                    const reason = isNatsPermission
-                        ? `Permission denied. You need one of these capabilities: ${ctx.callerCapabilities.join(", ")}`
-                        : message;
-                    return err(new UnexpectedError({
-                        cause,
-                        context: {
-                            method,
-                            subject,
-                            reason,
-                            requiredCapabilities: ctx.callerCapabilities,
-                            noResponders: isNoResponders,
-                        },
-                    }));
-                }
-                // Should be unreachable, but TypeScript needs explicit return
-                return err(new UnexpectedError({
-                    context: { method, subject, reason: "retry loop exhausted" },
-                }));
-            };
-            const msgResult = await requestWithRetry();
-            const m = msgResult.take();
-            if (isErr(m)) {
-                return m;
-            }
-            if (m.headers?.get("status") === "error") {
-                const json = safeJson(m).take();
-                if (isErr(json)) {
-                    return json;
-                }
-                const errorData = parse(TrellisErrorDataSchema, json).take();
-                if (isErr(errorData)) {
-                    return errorData;
-                }
-                const declaredErrorTypes = Array.isArray(ctx.declaredErrorTypes)
-                    ? ctx.declaredErrorTypes.filter((value) => typeof value === "string")
-                    : ctx.errors;
-                const runtimeErrors = Array.isArray(ctx.runtimeErrors)
-                    ? ctx.runtimeErrors.filter(isRuntimeRpcErrorDesc)
-                    : undefined;
-                const reconstructed = reconstructDeclaredRpcError(declaredErrorTypes, runtimeErrors, errorData, json);
-                if (reconstructed) {
-                    return err(reconstructed);
-                }
-                return err(new RemoteError({ error: errorData }));
-            }
-            const json = safeJson(m).take();
-            if (isErr(json)) {
-                return json;
-            }
-            const outputResult = parseRuntimeSchema(ctx.output, json).take();
-            if (isErr(outputResult)) {
-                return err(outputResult.error);
-            }
-            return ok(outputResult);
-        };
-        return withSpanAsync(span, async () => {
-            try {
-                const result = await attempt();
-                const value = result.take();
-                if (isErr(value)) {
-                    span.setStatus({
-                        code: SpanStatusCode.ERROR,
-                        message: value.error.message,
-                    });
-                }
-                else {
-                    span.setStatus({ code: SpanStatusCode.OK });
-                }
-                return result;
-            }
-            catch (cause) {
-                const unexpected = new UnexpectedError({ cause });
-                span.setStatus({
-                    code: SpanStatusCode.ERROR,
-                    message: unexpected.message,
-                });
-                span.recordException(unexpected);
-                return err(unexpected);
-            }
-            finally {
-                span.end();
-            }
-        });
-    }
-    async requestOrThrow(method, input, opts) {
-        const result = await this.request(method, input, opts);
-        const value = result.take();
-        if (isErr(value)) {
-            throw value.error;
-        }
-        return value;
+        return __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestBuiltRpcUnknown).call(this, method, input, ctx, opts);
     }
     operation(operation) {
         const descriptor = this.api["operations"]?.[operation];
@@ -515,19 +669,22 @@ export class Trellis {
         const transport = {
             requestJson: (subject, body) => __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestJson).call(this, subject, body),
             watchJson: (subject, body) => __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_watchJson).call(this, subject, body),
-            putTransfer: async (grant, body) => {
+            putTransfer: (grant, body) => AsyncResult.from((async () => {
                 const handle = createTransferHandle(this.nats, this.auth, this.timeout, grant);
-                if (!(handle instanceof Object) || !("put" in handle)) {
+                if (!(handle instanceof Object) || !("send" in handle)) {
                     return err(new TransferError({
                         operation: "transfer",
                         context: { reason: "invalid_operation_transfer_grant" },
                     }));
                 }
-                return await handle.put(body);
-            },
+                return await handle.send(body);
+            })()),
         };
         return new OperationInvoker(transport, descriptor);
     }
+    transfer(grant) {
+        return createTransferHandle(this.nats, this.auth, this.timeout, grant);
+    }
     /*
      * Mount a handler to process requests made to a specific Trellis API
      */
@@ -543,81 +700,94 @@ export class Trellis {
     respondWithError(msg, error) {
         __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_respondWithError).call(this, msg, error);
     }
-    async publish(event, data) {
-        const eventName = event;
-        const ctx = this.api["events"][eventName];
-        if (!ctx) {
-            return err(new UnexpectedError({
-                cause: __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_unknownApiError).call(this, "event", event.toString()),
-                context: { event: event.toString() },
-            }));
-        }
-        const subject = this.template(ctx.subject, data).take();
-        if (isErr(subject)) {
-            logger.error({ err: subject.error }, "Failed to template event.");
-            return subject;
-        }
-        const payload = {
-            ...data,
-            header: {
-                id: ulid(),
-                time: new Date().toISOString(),
-            },
-        };
-        const msg = encodeSchema(ctx.event, payload).take();
-        if (isErr(msg)) {
-            logger.error({ err: msg.error }, "Failed to encode event.");
-            return err(new UnexpectedError({ cause: msg.error }));
-        }
-        logger.trace({ subject }, `Publishing ${event.toString()} event.`);
-        await this.js.publish(subject, msg);
-        return ok(undefined);
+    publish(event, data) {
+        return AsyncResult.from((async () => {
+            try {
+                const eventName = event;
+                const ctx = this.api["events"][eventName];
+                if (!ctx) {
+                    return err(new UnexpectedError({
+                        cause: __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_unknownApiError).call(this, "event", event.toString()),
+                        context: { event: event.toString() },
+                    }));
+                }
+                const subject = this.template(ctx.subject, data).take();
+                if (isErr(subject)) {
+                    logger.error({ err: subject.error }, "Failed to template event.");
+                    return subject;
+                }
+                const payload = {
+                    ...data,
+                    header: {
+                        id: ulid(),
+                        time: new Date().toISOString(),
+                    },
+                };
+                const msg = encodeSchema(ctx.event, payload).take();
+                if (isErr(msg)) {
+                    logger.error({ err: msg.error }, "Failed to encode event.");
+                    return err(new UnexpectedError({ cause: msg.error }));
+                }
+                logger.trace({ subject }, `Publishing ${event.toString()} event.`);
+                await this.js.publish(subject, msg);
+                return ok(undefined);
+            }
+            catch (cause) {
+                return err(new UnexpectedError({ cause, context: { event: event.toString() } }));
+            }
+        })());
     }
-    async event(event, subjectData, fn, opts) {
-        const eventName = event;
-        const ctx = this.api["events"][eventName];
-        if (!ctx) {
-            return err(new UnexpectedError({
-                cause: __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_unknownApiError).call(this, "event", event.toString()),
-                context: { event: event.toString() },
-            }));
-        }
-        const subject = this.template(ctx.subject, subjectData, true).take();
-        if (isErr(subject))
-            return subject;
-        if (opts?.mode === "ephemeral") {
-            return __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_startEphemeralEvent).call(this, eventName, ctx, subject, fn, opts.signal);
-        }
-        const jsm = await jetstreamManager(this.nats);
-        const consumerName = opts?.durableName ??
-            `${this.name}-${event.replaceAll(".", "_")}`;
-        const addResult = await AsyncResult.try(() => jsm.consumers.add(this.stream, {
-            durable_name: consumerName,
-            ack_policy: "explicit",
-            deliver_policy: opts?.replay === "new" ? "new" : "all",
-            filter_subjects: [subject],
-        }));
-        // If add failed (consumer already exists), try to get existing consumer info
-        const consumerInfoResult = addResult.isOk()
-            ? addResult
-            : await AsyncResult.try(() => jsm.consumers.info(this.stream, consumerName));
-        const info = consumerInfoResult.take();
-        if (isErr(info))
-            return info;
-        const consumer = this.js.consumers.getConsumerFromInfo(info);
-        const messages = await consumer.consume();
-        if (opts?.signal) {
-            if (opts.signal.aborted) {
-                messages.stop();
+    event(event, subjectData, fn, opts) {
+        return AsyncResult.from((async () => {
+            try {
+                const eventName = event;
+                const ctx = this.api["events"][eventName];
+                if (!ctx) {
+                    return err(new UnexpectedError({
+                        cause: __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_unknownApiError).call(this, "event", event.toString()),
+                        context: { event: event.toString() },
+                    }));
+                }
+                const subject = this.template(ctx.subject, subjectData, true).take();
+                if (isErr(subject))
+                    return subject;
+                if (opts?.mode === "ephemeral") {
+                    return __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_startEphemeralEvent).call(this, eventName, ctx, subject, fn, opts.signal);
+                }
+                const jsm = await jetstreamManager(this.nats);
+                const consumerName = opts?.durableName ??
+                    `${this.name}-${String(event).replaceAll(".", "_")}`;
+                const addResult = await AsyncResult.try(() => jsm.consumers.add(this.stream, {
+                    durable_name: consumerName,
+                    ack_policy: "explicit",
+                    deliver_policy: opts?.replay === "new" ? "new" : "all",
+                    filter_subjects: [subject],
+                }));
+                const consumerInfoResult = addResult.isOk()
+                    ? addResult
+                    : await AsyncResult.try(() => jsm.consumers.info(this.stream, consumerName));
+                const info = consumerInfoResult.take();
+                if (isErr(info))
+                    return info;
+                const consumer = this.js.consumers.getConsumerFromInfo(info);
+                const messages = await consumer.consume();
+                if (opts?.signal) {
+                    if (opts.signal.aborted) {
+                        messages.stop();
+                    }
+                    else {
+                        opts.signal.addEventListener("abort", () => messages.stop(), {
+                            once: true,
+                        });
+                    }
+                }
+                __classPrivateFieldGet(this, _Trellis_tasks, "f").add(`event:${eventName}:${ulid()}`, __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_handleDurableEvent).call(this, eventName, ctx, messages, fn));
+                return ok(undefined);
             }
-            else {
-                opts.signal.addEventListener("abort", () => messages.stop(), {
-                    once: true,
-                });
+            catch (cause) {
+                return err(new UnexpectedError({ cause, context: { event: event.toString() } }));
             }
-        }
-        __classPrivateFieldGet(this, _Trellis_tasks, "f").add(`event:${eventName}:${ulid()}`, __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_handleDurableEvent).call(this, eventName, ctx, messages, fn));
-        return ok(undefined);
+        })());
     }
     wait() {
         return __classPrivateFieldGet(this, _Trellis_tasks, "f").wait();
@@ -653,18 +823,211 @@ export class Trellis {
         });
         return ok(result);
     }
-    async requestAuthValidate(input) {
+    requestAuthValidate(input) {
         const request = this.request.bind(this);
-        const result = await request("Auth.ValidateRequest", input);
-        return result;
+        return request("Auth.ValidateRequest", input);
     }
 }
-_Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplicitApi = new WeakMap(), _Trellis_noResponderMaxRetries = new WeakMap(), _Trellis_noResponderRetryMs = new WeakMap(), _Trellis_authBypassMethods = new WeakMap(), _Trellis_operationStore = new WeakMap(), _Trellis_instances = new WeakSet(), _Trellis_unknownApiError = function _Trellis_unknownApiError(kind, name) {
+_Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplicitApi = new WeakMap(), _Trellis_noResponderMaxRetries = new WeakMap(), _Trellis_noResponderRetryMs = new WeakMap(), _Trellis_authBypassMethods = new WeakMap(), _Trellis_onSessionNotFound = new WeakMap(), _Trellis_operationStore = new WeakMap(), _Trellis_instances = new WeakSet(), _Trellis_createStateFacade = function _Trellis_createStateFacade(state) {
+    const stores = (state ?? {});
+    const facade = Object.fromEntries(Object.entries(stores).map(([store, descriptor]) => {
+        if (descriptor.kind === "value") {
+            const client = {
+                get: () => AsyncResult.from((async () => {
+                    const result = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestBuiltRpc).call(this, "State.Get", { store }, STATE_RUNTIME_RPC.get);
+                    if (result.isErr())
+                        return result;
+                    return validateStateGetResult(descriptor, result.unwrapOrElse(() => {
+                        throw new Error("state get unexpectedly failed");
+                    }));
+                })()),
+                put: (value, opts) => AsyncResult.from((async () => {
+                    const encoded = encodeRuntimeSchema(descriptor.schema, value)
+                        .take();
+                    if (isErr(encoded)) {
+                        return Result.err(encoded.error);
+                    }
+                    const result = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestBuiltRpc).call(this, "State.Put", { store, value, ...opts }, STATE_RUNTIME_RPC.put);
+                    if (result.isErr())
+                        return result;
+                    return validateStatePutResult(descriptor, result.unwrapOrElse(() => {
+                        throw new Error("state put unexpectedly failed");
+                    }));
+                })()),
+                delete: (opts) => __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestBuiltRpc).call(this, "State.Delete", { store, ...opts }, STATE_RUNTIME_RPC.delete),
+            };
+            return [store, client];
+        }
+        const mapClient = (prefix) => ({
+            get: (key) => AsyncResult.from((async () => {
+                const result = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestBuiltRpc).call(this, "State.Get", { store, key: joinStatePath(prefix, key) }, STATE_RUNTIME_RPC.get);
+                if (result.isErr())
+                    return result;
+                return validateStateGetResult(descriptor, result.unwrapOrElse(() => {
+                    throw new Error("state get unexpectedly failed");
+                }));
+            })()),
+            put: (key, value, opts) => AsyncResult.from((async () => {
+                const encoded = encodeRuntimeSchema(descriptor.schema, value)
+                    .take();
+                if (isErr(encoded)) {
+                    return Result.err(encoded.error);
+                }
+                const result = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestBuiltRpc).call(this, "State.Put", { store, key: joinStatePath(prefix, key), value, ...opts }, STATE_RUNTIME_RPC.put);
+                if (result.isErr())
+                    return result;
+                return validateStatePutResult(descriptor, result.unwrapOrElse(() => {
+                    throw new Error("state put unexpectedly failed");
+                }));
+            })()),
+            delete: (key, opts) => __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestBuiltRpc).call(this, "State.Delete", { store, key: joinStatePath(prefix, key), ...opts }, STATE_RUNTIME_RPC.delete),
+            list: (opts) => AsyncResult.from((async () => {
+                const result = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestBuiltRpc).call(this, "State.List", {
+                    store,
+                    ...(prefix ? { prefix } : {}),
+                    offset: opts?.offset ?? 0,
+                    limit: opts?.limit ?? DEFAULT_STATE_LIST_LIMIT,
+                }, STATE_RUNTIME_RPC.list);
+                if (result.isErr())
+                    return result;
+                return validateStateListResult(descriptor, result.unwrapOrElse(() => {
+                    throw new Error("state list unexpectedly failed");
+                }));
+            })()),
+            prefix: (path) => mapClient(joinStatePath(prefix, path)),
+        });
+        return [store, mapClient()];
+    }));
+    return facade;
+}, _Trellis_unknownApiError = function _Trellis_unknownApiError(kind, name) {
     const base = `Unknown ${kind} '${name}'.`;
     if (__classPrivateFieldGet(this, _Trellis_hasExplicitApi, "f")) {
         return new Error(`${base} Did you forget to include its API module?`);
     }
     return new Error(`${base} No API surface was provided. Pass opts.api, use createClient(contract, ...), or await createCoreClient(...) instead.`);
+}, _Trellis_requestBuiltRpcUnknown = function _Trellis_requestBuiltRpcUnknown(method, input, ctx, opts) {
+    return __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestBuiltRpc).call(this, method, input, ctx, opts);
+}, _Trellis_requestBuiltRpc = function _Trellis_requestBuiltRpc(method, input, ctx, opts) {
+    return AsyncResult.from((async () => {
+        __classPrivateFieldGet(this, _Trellis_log, "f").trace({ method: String(method) }, `Calling ${method.toString()}.`);
+        const msg = encodeRuntimeSchema(ctx.input, input).take();
+        if (isErr(msg)) {
+            return msg;
+        }
+        const subject = this.template(ctx.subject, input).take();
+        if (isErr(subject)) {
+            return subject;
+        }
+        const span = startClientSpan(method, subject);
+        const attempt = async () => {
+            const proof = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_createProof).call(this, subject, msg);
+            const headers = natsHeaders();
+            headers.set("session-key", this.auth.sessionKey);
+            headers.set("proof", proof);
+            injectTraceContext(createNatsHeaderCarrier(headers), span);
+            const msgResult = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestMessageWithRetry).call(this, {
+                method,
+                subject,
+                payload: msg,
+                headers,
+                timeout: opts?.timeout ?? this.timeout,
+                callerCapabilities: ctx.callerCapabilities,
+            });
+            const response = msgResult.take();
+            if (isErr(response)) {
+                return response;
+            }
+            if (response.headers?.get("status") === "error") {
+                const json = safeJson(response).take();
+                if (isErr(json)) {
+                    return err(requestFailedTransportError({
+                        code: "trellis.request.invalid_response",
+                        message: "Trellis returned an invalid response.",
+                        hint: "Retry the request. If it keeps happening, check the Trellis capability handling this request.",
+                        method,
+                        subject,
+                        cause: json.error.cause,
+                    }));
+                }
+                const errorData = parse(TrellisErrorDataSchema, json).take();
+                if (isErr(errorData)) {
+                    return err(requestFailedTransportError({
+                        code: "trellis.request.invalid_response",
+                        message: "Trellis returned an invalid response.",
+                        hint: "Retry the request. If it keeps happening, check the Trellis capability handling this request.",
+                        method,
+                        subject,
+                        cause: errorData.error,
+                    }));
+                }
+                const declaredErrorTypes = Array.isArray(ctx.declaredErrorTypes)
+                    ? ctx.declaredErrorTypes.filter((value) => typeof value === "string")
+                    : ctx.errors;
+                const runtimeErrors = Array.isArray(ctx.runtimeErrors)
+                    ? ctx.runtimeErrors.filter(isRuntimeRpcErrorDesc)
+                    : undefined;
+                const reconstructed = reconstructDeclaredRpcError(declaredErrorTypes, runtimeErrors, errorData, json);
+                if (reconstructed) {
+                    await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_handleSessionNotFound).call(this, reconstructed);
+                    return err(reconstructed);
+                }
+                const remoteError = new RemoteError({ error: errorData });
+                await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_handleSessionNotFound).call(this, remoteError);
+                return err(remoteError);
+            }
+            const json = safeJson(response).take();
+            if (isErr(json)) {
+                return err(requestFailedTransportError({
+                    code: "trellis.request.invalid_response",
+                    message: "Trellis returned an invalid response.",
+                    hint: "Retry the request. If it keeps happening, check the Trellis capability handling this request.",
+                    method,
+                    subject,
+                    cause: json.error.cause,
+                }));
+            }
+            const outputResult = parseRuntimeSchema(ctx.output, json).take();
+            if (isErr(outputResult)) {
+                return err(outputResult.error);
+            }
+            return ok(outputResult);
+        };
+        return await withSpanAsync(span, async () => {
+            try {
+                const result = await attempt();
+                const value = result.take();
+                if (isErr(value)) {
+                    span.setStatus({
+                        code: SpanStatusCode.ERROR,
+                        message: value.error.message,
+                    });
+                }
+                else {
+                    span.setStatus({ code: SpanStatusCode.OK });
+                }
+                return result;
+            }
+            catch (cause) {
+                const unexpected = cause instanceof TransportError
+                    ? cause
+                    : new UnexpectedError({ cause });
+                span.setStatus({
+                    code: SpanStatusCode.ERROR,
+                    message: unexpected.message,
+                });
+                span.recordException(unexpected);
+                return err(unexpected);
+            }
+            finally {
+                span.end();
+            }
+        });
+    })());
+}, _Trellis_handleSessionNotFound = async function _Trellis_handleSessionNotFound(error) {
+    if (!__classPrivateFieldGet(this, _Trellis_onSessionNotFound, "f") || !isTransientAuthValidateSessionError(error)) {
+        return;
+    }
+    await __classPrivateFieldGet(this, _Trellis_onSessionNotFound, "f").call(this);
 }, _Trellis_handleRPC = function _Trellis_handleRPC(method, fn, subjectData = {}) {
     // Get API details
     const ctx = this.api["rpc"][method];
@@ -757,7 +1120,7 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     return crypto.subtle.verify({ name: "Ed25519" }, pub, toArrayBuffer(base64urlDecode(proof)), toArrayBuffer(digest));
                 });
                 const signatureOk = verifyResult.isOk() &&
-                    (await verifyResult).take() === true;
+                    verifyResult.take() === true;
                 if (!signatureOk) {
                     span.setStatus({
                         code: SpanStatusCode.ERROR,
@@ -770,14 +1133,13 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 }
                 let auth;
                 for (let attempt = 0; attempt < DEFAULT_AUTH_VALIDATE_SESSION_RETRY_ATTEMPTS; attempt++) {
-                    const authResult = await this.requestAuthValidate({
+                    const authValue = await this.requestAuthValidate({
                         sessionKey,
                         proof,
                         subject: msg.subject,
                         payloadHash: base64urlEncode(payloadHash),
                         capabilities: [...ctx.callerCapabilities],
-                    });
-                    const authValue = authResult.take();
+                    }).take();
                     if (!isErr(authValue)) {
                         auth = authValue;
                         break;
@@ -849,13 +1211,17 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             }
             if (caller.type === "device") {
                 span.setAttribute("device.id", caller.deviceId);
-                span.setAttribute("device.profile_id", caller.profileId);
+                span.setAttribute("device.deployment_id", caller.deploymentId);
             }
             const invokeHandler = fn;
-            const handlerResultWrapped = await AsyncResult.try(async () => await Promise.resolve(invokeHandler(parsedInput, {
-                caller,
-                sessionKey: callerSessionKey,
-            }, handlerTrellis)));
+            const handlerResultWrapped = await AsyncResult.try(async () => await Promise.resolve(invokeHandler({
+                input: parsedInput,
+                context: {
+                    caller,
+                    sessionKey: callerSessionKey,
+                },
+                trellis: handlerTrellis,
+            })));
             if (handlerResultWrapped.isErr()) {
                 const error = handlerResultWrapped.error.withContext({ method });
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({
@@ -872,7 +1238,7 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 span.recordException(error);
                 return err(error);
             }
-            const handlerResult = (await handlerResultWrapped).take();
+            const handlerResult = handlerResultWrapped.take();
             const handlerOutcome = handlerResult.take();
             if (isErr(handlerOutcome)) {
                 const handlerError = handlerOutcome.error;
@@ -999,51 +1365,122 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
     const digest = await sha256(input);
     const sigBytes = await this.auth.sign(digest);
     return base64urlEncode(sigBytes);
-}, _Trellis_requestJson = async function _Trellis_requestJson(subject, body) {
-    const payload = JSON.stringify(body);
-    const proof = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_createProof).call(this, subject, payload);
-    const headers = natsHeaders();
-    headers.set("session-key", this.auth.sessionKey);
-    headers.set("proof", proof);
-    const response = await AsyncResult.try(() => this.nats.request(subject, payload, {
-        timeout: this.timeout,
-        headers,
-    })).take();
-    if (isErr(response)) {
-        return response;
+}, _Trellis_requestMessageWithRetry = async function _Trellis_requestMessageWithRetry(args) {
+    for (let retry = 0; retry <= __classPrivateFieldGet(this, _Trellis_noResponderMaxRetries, "f"); retry++) {
+        const result = await AsyncResult.try(() => this.nats.request(args.subject, args.payload, {
+            headers: args.headers,
+            timeout: args.timeout,
+        }));
+        if (result.isOk()) {
+            return ok(result.take());
+        }
+        const cause = result.error.cause;
+        const message = cause instanceof Error ? cause.message : String(cause);
+        const isNoResponders = message.includes("no responders");
+        if (isNoResponders && retry < __classPrivateFieldGet(this, _Trellis_noResponderMaxRetries, "f")) {
+            __classPrivateFieldGet(this, _Trellis_log, "f").debug({ method: args.method, subject: args.subject, retry }, "No responders, retrying...");
+            await new Promise((resolve) => setTimeout(resolve, __classPrivateFieldGet(this, _Trellis_noResponderRetryMs, "f") * (retry + 1)));
+            continue;
+        }
+        __classPrivateFieldGet(this, _Trellis_log, "f").warn({ method: args.method, subject: args.subject, error: message }, "NATS request failed");
+        return err(classifyRequestTransportFailure({
+            method: args.method,
+            subject: args.subject,
+            callerCapabilities: args.callerCapabilities,
+            cause,
+        }));
     }
-    return safeJson(response);
-}, _Trellis_watchJson = async function _Trellis_watchJson(subject, body) {
-    const payload = JSON.stringify(body);
-    const proof = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_createProof).call(this, subject, payload);
-    const headers = natsHeaders();
-    headers.set("session-key", this.auth.sessionKey);
-    headers.set("proof", proof);
-    const inbox = createInbox(`_INBOX.${this.auth.sessionKey.slice(0, 16)}`);
-    const sub = this.nats.subscribe(inbox);
-    try {
-        this.nats.publish(subject, payload, {
+    return err(requestFailedTransportError({
+        code: "trellis.request.retry_exhausted",
+        message: "Trellis could not complete the request after retrying.",
+        hint: "Retry the request. If it keeps failing, check that the target service is available.",
+        method: args.method,
+        subject: args.subject,
+        context: { retries: __classPrivateFieldGet(this, _Trellis_noResponderMaxRetries, "f") + 1 },
+    }));
+}, _Trellis_requestJson = function _Trellis_requestJson(subject, body) {
+    return AsyncResult.from((async () => {
+        const payload = JSON.stringify(body);
+        const proof = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_createProof).call(this, subject, payload);
+        const headers = natsHeaders();
+        headers.set("session-key", this.auth.sessionKey);
+        headers.set("proof", proof);
+        const response = (await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestMessageWithRetry).call(this, {
+            subject,
+            payload,
             headers,
-            reply: inbox,
-        });
-        await this.nats.flush();
-    }
-    catch (cause) {
-        sub.unsubscribe();
-        return err(new UnexpectedError({ cause }));
-    }
-    return ok((async function* () {
+            timeout: this.timeout,
+        })).take();
+        if (isErr(response)) {
+            return response;
+        }
+        const json = safeJson(response).take();
+        if (isErr(json)) {
+            return err(createTransportError({
+                code: "trellis.request.invalid_response",
+                message: "Trellis returned an invalid response.",
+                hint: "Retry the request. If it keeps happening, reconnect to Trellis and try again.",
+                cause: json.error.cause,
+                context: { subject },
+            }));
+        }
+        return ok(json);
+    })());
+}, _Trellis_watchJson = function _Trellis_watchJson(subject, body) {
+    return AsyncResult.from((async () => {
+        const payload = JSON.stringify(body);
+        const proof = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_createProof).call(this, subject, payload);
+        const headers = natsHeaders();
+        headers.set("session-key", this.auth.sessionKey);
+        headers.set("proof", proof);
+        const inbox = createInbox(`_INBOX.${this.auth.sessionKey.slice(0, 16)}`);
+        const sub = this.nats.subscribe(inbox);
         try {
-            for await (const msg of sub) {
-                if (msg.headers?.get("status") === "error") {
-                    yield err(new UnexpectedError({ cause: new Error(msg.string()) }));
-                    continue;
-                }
-                yield safeJson(msg);
-            }
+            this.nats.publish(subject, payload, {
+                headers,
+                reply: inbox,
+            });
+            await this.nats.flush();
         }
-        finally {
+        catch (cause) {
             sub.unsubscribe();
+            return err(createTransportError({
+                code: "trellis.watch.failed",
+                message: "Trellis could not start the operation watch.",
+                hint: "Retry watching the operation. If it keeps failing, reconnect to Trellis and try again.",
+                cause,
+                context: { subject },
+            }));
         }
+        return ok((async function* () {
+            try {
+                for await (const msg of sub) {
+                    if (msg.headers?.get("status") === "error") {
+                        yield err(createTransportError({
+                            code: "trellis.watch.failed",
+                            message: "Trellis stopped the operation watch.",
+                            hint: "Retry watching the operation. If it keeps happening, reconnect to Trellis and try again.",
+                            context: { subject, frame: msg.string() },
+                        }));
+                        continue;
+                    }
+                    const json = safeJson(msg).take();
+                    if (isErr(json)) {
+                        yield err(createTransportError({
+                            code: "trellis.watch.invalid_response",
+                            message: "Trellis returned an invalid watch update.",
+                            hint: "Retry watching the operation. If it keeps happening, reconnect to Trellis and try again.",
+                            cause: json.error.cause,
+                            context: { subject },
+                        }));
+                        continue;
+                    }
+                    yield ok(json);
+                }
+            }
+            finally {
+                sub.unsubscribe();
+            }
+        })());
     })());
 };