@qlever-llc/trellis 0.10.10 → 0.10.12

package/esm/trellis.js

@@ -9,12 +9,12 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
     if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
     return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
 };
-var _Trellis_instances, _Trellis_log, _Trellis_tasks, _Trellis_hasExplicitApi, _Trellis_noResponderMaxRetries, _Trellis_noResponderRetryMs, _Trellis_onSessionNotFound, _Trellis_operationStore, _Trellis_eventConsumers, _Trellis_durableEventLoops, _Trellis_createStateFacade, _Trellis_createRpcFacade, _Trellis_createRpcHandleFacade, _Trellis_createEventFacade, _Trellis_createEventPublishFacade, _Trellis_createFeedFacade, _Trellis_createHandlerTrellis, _Trellis_createOperationFacade, _Trellis_unknownApiError, _Trellis_requestBuiltRpcUnknown, _Trellis_requestBuiltRpc, _Trellis_handleBrowserAuthRequired, _Trellis_authenticateFeedRequest, _Trellis_subscribeFeed, _Trellis_handleFeed, _Trellis_processFeedMessage, _Trellis_handleRPC, _Trellis_processRPCMessage, _Trellis_respondWithPayload, _Trellis_respondWithError, _Trellis_startEphemeralEvent, _Trellis_resolveEventConsumerGroup, _Trellis_registerDurableEventHandler, _Trellis_startDurableEventConsumer, _Trellis_durableEventConsumerGroupReady, _Trellis_runDurableEventConsumer, _Trellis_handleDurableEvent, _Trellis_handleDurableEventConsumer, _Trellis_parseEventMessage, _Trellis_escapeSubjectToken, _Trellis_currentIat, _Trellis_createProof, _Trellis_requestMessageWithRetry, _Trellis_requestJson, _Trellis_watchJson;
+var _Trellis_instances, _Trellis_log, _Trellis_tasks, _Trellis_hasExplicitApi, _Trellis_noResponderMaxRetries, _Trellis_noResponderRetryMs, _Trellis_onSessionNotFound, _Trellis_operationStore, _Trellis_eventConsumers, _Trellis_durableEventLoops, _Trellis_createStateFacade, _Trellis_createRpcFacade, _Trellis_createRpcHandleFacade, _Trellis_createEventFacade, _Trellis_createEventPublishFacade, _Trellis_createFeedFacade, _Trellis_createHandlerTrellis, _Trellis_createOperationFacade, _Trellis_unknownApiError, _Trellis_requestBuiltRpcUnknown, _Trellis_requestBuiltRpc, _Trellis_handleBrowserAuthRequired, _Trellis_authenticateFeedRequest, _Trellis_subscribeFeed, _Trellis_handleFeed, _Trellis_processFeedMessage, _Trellis_handleRPC, _Trellis_processRPCMessage, _Trellis_respondWithPayload, _Trellis_respondWithError, _Trellis_startEphemeralEvent, _Trellis_invokeEventHandler, _Trellis_resolveEventConsumerGroup, _Trellis_registerDurableEventHandler, _Trellis_startDurableEventConsumer, _Trellis_durableEventConsumerGroupReady, _Trellis_runDurableEventConsumer, _Trellis_handleDurableEvent, _Trellis_handleDurableEventConsumer, _Trellis_parseEventMessage, _Trellis_escapeSubjectToken, _Trellis_currentIat, _Trellis_createProof, _Trellis_requestMessageWithRetry, _Trellis_requestJson, _Trellis_watchJson;
 import { jetstream, jetstreamManager, } from "@nats-io/jetstream";
 import { createInbox, headers as natsHeaders, } from "@nats-io/nats-core";
 import { CONTRACT_JOBS_METADATA, CONTRACT_KV_METADATA, CONTRACT_STATE_METADATA, } from "./contract_support/mod.js";
 import { AsyncResult, BaseError, err, isErr, ok, Result, } from "@qlever-llc/result";
-import { createNatsHeaderCarrier, extractTraceContext, injectTraceContext, SpanStatusCode, startClientSpan, startServerSpan, withSpanAsync, } from "./tracing.js";
+import { createNatsHeaderCarrier, extractTraceContext, injectTraceContext, recordTrellisError, SpanStatusCode, startClientSpan, startServerSpan, withSpanAsync, } from "./telemetry/mod.js";
 import { Type } from "typebox";
 import { Pointer } from "typebox/value";
 import { ulid } from "ulid";
@@ -190,6 +190,27 @@ export function isOperationDeferred(value) {
 export function isResultLike(value) {
     return value instanceof Result;
 }
+function compactHandlerErrorContext(context) {
+    return Object.fromEntries(Object.entries(context).filter(([key, value]) => key !== "traceId" && value !== undefined));
+}
+function sanitizeHandlerErrorContext(error) {
+    delete error.getContext().subject;
+}
+export function annotateHandlerBoundaryError(cause, context) {
+    const error = cause instanceof BaseError && !(cause instanceof RemoteError)
+        ? cause
+        : new UnexpectedError({ cause });
+    sanitizeHandlerErrorContext(error);
+    error.withContext(compactHandlerErrorContext(context));
+    error.withTraceId(context.traceId);
+    return error;
+}
+function recordRuntimeError(error, attributes) {
+    recordTrellisError(error, {
+        messagingSystem: "nats",
+        ...attributes,
+    });
+}
 const DurableOperationSignalSchema = Type.Object({
     operationId: Type.String(),
     sequence: Type.Number(),
@@ -220,8 +241,11 @@ const DurableOperationSnapshotSchema = Type.Object({
     })),
     output: Type.Optional(Type.Any()),
     error: Type.Optional(Type.Object({
+        id: Type.Optional(Type.String()),
         type: Type.String(),
         message: Type.String(),
+        context: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
+        traceId: Type.Optional(Type.String()),
     })),
 });
 export const DurableOperationRecordSchema = Type.Object({
@@ -727,6 +751,18 @@ export class Trellis {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "contractId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "contractDigest", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "nats", {
             enumerable: true,
             configurable: true,
@@ -769,6 +805,8 @@ export class Trellis {
         __classPrivateFieldSet(this, _Trellis_log, (opts?.log ?? logger).child({ lib: "trellis" }), "f");
         this.timeout = opts?.timeout ?? 3000;
         this.stream = opts?.stream ?? "trellis";
+        this.contractId = opts?.contractId;
+        this.contractDigest = opts?.contractDigest;
         __classPrivateFieldSet(this, _Trellis_hasExplicitApi, api !== undefined, "f");
         __classPrivateFieldSet(this, _Trellis_noResponderMaxRetries, opts?.noResponderRetry?.maxAttempts ??
             DEFAULT_NO_RESPONDER_MAX_RETRIES, "f");
@@ -903,14 +941,27 @@ export class Trellis {
             const eventName = event;
             const ctx = this.api["events"][eventName];
             if (!ctx) {
-                return err(new UnexpectedError({
+                const error = new UnexpectedError({
                     cause: __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_unknownApiError).call(this, "event", event.toString()),
                     context: { event: event.toString() },
-                }));
+                });
+                recordRuntimeError(error, {
+                    surface: "event",
+                    direction: "publisher",
+                    operation: event,
+                    phase: "prepare",
+                });
+                return err(error);
             }
             const subject = this.template(ctx.subject, data).take();
             if (isErr(subject)) {
                 logger.error({ err: subject.error }, "Failed to template event.");
+                recordRuntimeError(subject.error, {
+                    surface: "event",
+                    direction: "publisher",
+                    operation: event,
+                    phase: "request_encoding",
+                });
                 return subject;
             }
             const header = {
@@ -924,7 +975,14 @@ export class Trellis {
             const msg = encodeSchema(ctx.event, payload).take();
             if (isErr(msg)) {
                 logger.error({ err: msg.error }, "Failed to encode event.");
-                return err(new UnexpectedError({ cause: msg.error }));
+                const error = new UnexpectedError({ cause: msg.error });
+                recordRuntimeError(error, {
+                    surface: "event",
+                    direction: "publisher",
+                    operation: event,
+                    phase: "request_encoding",
+                });
+                return err(error);
             }
             const headers = natsHeaders();
             headers.set("Nats-Msg-Id", header.id);
@@ -942,7 +1000,17 @@ export class Trellis {
             }));
         }
         catch (cause) {
-            return err(new UnexpectedError({ cause, context: { event: event.toString() } }));
+            const error = new UnexpectedError({
+                cause,
+                context: { event: event.toString() },
+            });
+            recordRuntimeError(error, {
+                surface: "event",
+                direction: "publisher",
+                operation: event,
+                phase: "prepare",
+            });
+            return err(error);
         }
     }
     /**
@@ -961,7 +1029,17 @@ export class Trellis {
                 return ok(undefined);
             }
             catch (cause) {
-                return err(new UnexpectedError({ cause, context: { event: event.event } }));
+                const error = new UnexpectedError({
+                    cause,
+                    context: { event: event.event },
+                });
+                recordRuntimeError(error, {
+                    surface: "event",
+                    direction: "publisher",
+                    operation: event.event,
+                    phase: "publish",
+                });
+                return err(error);
             }
         })());
     }
@@ -1197,10 +1275,22 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         __classPrivateFieldGet(this, _Trellis_log, "f").trace({ method: String(method) }, `Calling ${method.toString()}.`);
         const msg = encodeRuntimeSchema(ctx.input, input).take();
         if (isErr(msg)) {
+            recordRuntimeError(msg.error, {
+                surface: "rpc",
+                direction: "client",
+                operation: method,
+                phase: "request_encoding",
+            });
             return msg;
         }
         const subject = this.template(ctx.subject, input).take();
         if (isErr(subject)) {
+            recordRuntimeError(subject.error, {
+                surface: "rpc",
+                direction: "client",
+                operation: method,
+                phase: "request_encoding",
+            });
             return subject;
         }
         const span = startClientSpan(method, subject);
@@ -1222,30 +1312,50 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             });
             const response = msgResult.take();
             if (isErr(response)) {
+                recordRuntimeError(response.error, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "request_send",
+                });
                 return response;
             }
             if (response.headers?.get("status") === "error") {
                 const json = safeJson(response).take();
                 if (isErr(json)) {
-                    return err(requestFailedTransportError({
+                    const error = requestFailedTransportError({
                         code: "trellis.request.invalid_response",
                         message: "Trellis returned an invalid response.",
                         hint: "Retry the request. If it keeps happening, check the Trellis capability handling this request.",
                         method,
                         subject,
                         cause: json.error.cause,
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "client",
+                        operation: method,
+                        phase: "response_decoding",
+                    });
+                    return err(error);
                 }
                 const errorData = parse(TrellisErrorDataSchema, json).take();
                 if (isErr(errorData)) {
-                    return err(requestFailedTransportError({
+                    const error = requestFailedTransportError({
                         code: "trellis.request.invalid_response",
                         message: "Trellis returned an invalid response.",
                         hint: "Retry the request. If it keeps happening, check the Trellis capability handling this request.",
                         method,
                         subject,
                         cause: errorData.error,
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "client",
+                        operation: method,
+                        phase: "response_decoding",
+                    });
+                    return err(error);
                 }
                 const declaredErrorTypes = Array.isArray(ctx.declaredErrorTypes)
                     ? ctx.declaredErrorTypes.filter((value) => typeof value === "string")
@@ -1256,25 +1366,50 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 const reconstructed = reconstructDeclaredRpcError(declaredErrorTypes, runtimeErrors, errorData, json);
                 if (reconstructed) {
                     await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_handleBrowserAuthRequired).call(this, reconstructed);
+                    recordRuntimeError(new RemoteError({ error: errorData }), {
+                        surface: "rpc",
+                        direction: "client",
+                        operation: method,
+                        phase: "remote_error",
+                    });
                     return err(reconstructed);
                 }
                 const remoteError = new RemoteError({ error: errorData });
                 await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_handleBrowserAuthRequired).call(this, remoteError);
+                recordRuntimeError(remoteError, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "remote_error",
+                });
                 return err(remoteError);
             }
             const json = safeJson(response).take();
             if (isErr(json)) {
-                return err(requestFailedTransportError({
+                const error = requestFailedTransportError({
                     code: "trellis.request.invalid_response",
                     message: "Trellis returned an invalid response.",
                     hint: "Retry the request. If it keeps happening, check the Trellis capability handling this request.",
                     method,
                     subject,
                     cause: json.error.cause,
-                }));
+                });
+                recordRuntimeError(error, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "response_decoding",
+                });
+                return err(error);
             }
             const outputResult = parseRuntimeSchema(ctx.output, json).take();
             if (isErr(outputResult)) {
+                recordRuntimeError(outputResult.error, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "response_decoding",
+                });
                 return err(outputResult.error);
             }
             return ok(outputResult);
@@ -1303,6 +1438,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     message: unexpected.message,
                 });
                 span.recordException(unexpected);
+                recordRuntimeError(unexpected, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "unexpected",
+                });
                 return err(unexpected);
             }
             finally {
@@ -1371,11 +1512,25 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
 }, _Trellis_subscribeFeed = function _Trellis_subscribeFeed(feed, descriptor, input, opts) {
     return AsyncResult.from((async () => {
         const payload = encodeRuntimeSchema(descriptor.input, input).take();
-        if (isErr(payload))
+        if (isErr(payload)) {
+            recordRuntimeError(payload.error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "request_encoding",
+            });
             return payload;
+        }
         const subject = this.template(descriptor.subject, input).take();
-        if (isErr(subject))
+        if (isErr(subject)) {
+            recordRuntimeError(subject.error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "request_template",
+            });
             return subject;
+        }
         const authHeaders = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_createProof).call(this, subject, payload);
         const headers = natsHeaders();
         headers.set("session-key", this.auth.sessionKey);
@@ -1395,13 +1550,20 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         catch (cause) {
             opts?.signal?.removeEventListener("abort", abort);
             sub.unsubscribe();
-            return err(createTransportError({
+            const error = createTransportError({
                 code: "trellis.feed.subscribe_failed",
                 message: "Trellis could not subscribe to the feed.",
                 hint: "Retry the subscription. If it keeps failing, check Trellis runtime health.",
                 cause,
                 context: { feed, subject },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "request_send",
+            });
+            return err(error);
         }
         let timeoutId;
         let abortHandler;
@@ -1427,7 +1589,7 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         if (firstFrame === "timeout" || firstFrame === "aborted") {
             opts?.signal?.removeEventListener("abort", abort);
             sub.unsubscribe();
-            return err(createTransportError({
+            const error = createTransportError({
                 code: firstFrame === "timeout"
                     ? "trellis.feed.subscribe_timeout"
                     : "trellis.feed.subscribe_aborted",
@@ -1438,28 +1600,49 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     ? "Check that the target service is running and has the current deployment digest, then retry."
                     : "Retry the subscription if the feed is still needed.",
                 context: { feed, subject },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "handshake",
+            });
+            return err(error);
         }
         if (firstFrame.done) {
             opts?.signal?.removeEventListener("abort", abort);
             sub.unsubscribe();
-            return err(createTransportError({
+            const error = createTransportError({
                 code: "trellis.feed.subscribe_closed",
                 message: "Trellis closed the feed before acknowledging it.",
                 hint: "Retry the subscription. If it keeps failing, check Trellis runtime health.",
                 context: { feed, subject },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "handshake",
+            });
+            return err(error);
         }
         const firstMessage = firstFrame.value;
         if (firstMessage.headers?.get("status") === "error") {
             opts?.signal?.removeEventListener("abort", abort);
             sub.unsubscribe();
-            return err(createTransportError({
+            const error = createTransportError({
                 code: "trellis.feed.failed",
                 message: "Trellis rejected the feed subscription.",
                 hint: "Retry the subscription. If it keeps failing, check Trellis runtime health and permissions.",
                 context: { feed, subject, frame: firstMessage.string() },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "remote_error",
+            });
+            return err(error);
         }
         const firstEvent = firstMessage.headers?.get("feed-status") === "ready"
             ? undefined
@@ -1469,19 +1652,40 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             try {
                 const parseFeedFrame = (msg) => {
                     if (msg.headers?.get("status") === "error") {
-                        throw createTransportError({
+                        const error = createTransportError({
                             code: "trellis.feed.failed",
                             message: "Trellis stopped the feed.",
                             hint: "Retry the subscription. If it keeps failing, check Trellis runtime health.",
                             context: { feed, subject, frame: msg.string() },
                         });
+                        recordRuntimeError(error, {
+                            surface: "feed",
+                            direction: "client",
+                            operation: feed,
+                            phase: "remote_error",
+                        });
+                        throw error;
                     }
                     const json = safeJson(msg).take();
-                    if (isErr(json))
+                    if (isErr(json)) {
+                        recordRuntimeError(json.error, {
+                            surface: "feed",
+                            direction: "client",
+                            operation: feed,
+                            phase: "event_decoding",
+                        });
                         throw json.error;
+                    }
                     const parsed = parseRuntimeSchema(eventSchema, json).take();
-                    if (isErr(parsed))
+                    if (isErr(parsed)) {
+                        recordRuntimeError(parsed.error, {
+                            surface: "feed",
+                            direction: "client",
+                            operation: feed,
+                            phase: "event_validation",
+                        });
                         throw parsed.error;
+                    }
                     return parsed;
                 };
                 if (firstEvent)
@@ -1509,13 +1713,20 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         await this.nats.flush();
     }
     catch (cause) {
-        throw createTransportError({
+        const error = createTransportError({
             code: "trellis.feed.listen_failed",
             message: "Trellis could not listen for feed requests.",
             hint: "Check the service deployment digest and runtime permissions, then restart the service.",
             cause,
             context: { feed, subject },
         });
+        recordRuntimeError(error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "listen",
+        });
+        throw error;
     }
     const task = AsyncResult.try(async () => {
         for await (const msg of sub) {
@@ -1528,9 +1739,20 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     }
                 }
                 catch (cause) {
-                    const error = cause instanceof BaseError
-                        ? cause
-                        : new UnexpectedError({ cause });
+                    const error = annotateHandlerBoundaryError(cause, {
+                        feed,
+                        requestId: msg.headers?.get("request-id"),
+                        service: this.name,
+                        contractId: this.contractId,
+                        contractDigest: this.contractDigest,
+                        traceId: traceIdFromTraceparent(msg.headers?.get("traceparent")),
+                    });
+                    recordRuntimeError(error, {
+                        surface: "feed",
+                        direction: "server",
+                        operation: feed,
+                        phase: "handler_throw",
+                    });
                     __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_respondWithError).call(this, msg, error);
                 }
             })();
@@ -1539,11 +1761,25 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
     __classPrivateFieldGet(this, _Trellis_tasks, "f").add(`feed:${feed}`, task);
 }, _Trellis_processFeedMessage = async function _Trellis_processFeedMessage(feed, descriptor, msg, handler) {
     const json = safeJson(msg).take();
-    if (isErr(json))
+    if (isErr(json)) {
+        recordRuntimeError(json.error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "request_decoding",
+        });
         return json;
+    }
     const parsed = parseRuntimeSchema(descriptor.input, json).take();
-    if (isErr(parsed))
+    if (isErr(parsed)) {
+        recordRuntimeError(parsed.error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "input_validation",
+        });
         return parsed;
+    }
     const caller = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_authenticateFeedRequest).call(this, {
         feed,
         subject: msg.subject,
@@ -1552,12 +1788,26 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         requiredCapabilities: descriptor.subscribeCapabilities,
     });
     const callerValue = caller.take();
-    if (isErr(callerValue))
+    if (isErr(callerValue)) {
+        recordRuntimeError(callerValue.error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "auth",
+        });
         return callerValue;
+    }
     if (!msg.reply) {
-        return err(new UnexpectedError({
+        const error = new UnexpectedError({
             context: { feed, reason: "missing_reply" },
-        }));
+        });
+        recordRuntimeError(error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "handshake",
+        });
+        return err(error);
     }
     const readyHeaders = natsHeaders();
     readyHeaders.set("feed-status", "ready");
@@ -1565,24 +1815,73 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
     await this.nats.flush();
     const controller = new AbortController();
     try {
-        await handler({
+        const handlerResult = await handler({
             input: parsed,
             caller: callerValue,
             signal: controller.signal,
             emit: (event) => AsyncResult.from((async () => {
                 const payload = encodeRuntimeSchema(descriptor.event, event).take();
-                if (isErr(payload))
+                if (isErr(payload)) {
+                    recordRuntimeError(payload.error, {
+                        surface: "feed",
+                        direction: "server",
+                        operation: feed,
+                        phase: "event_encoding",
+                    });
                     return payload;
+                }
                 if (!msg.reply) {
-                    return err(new UnexpectedError({
+                    const error = new UnexpectedError({
                         context: { feed, reason: "missing_reply" },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "feed",
+                        direction: "server",
+                        operation: feed,
+                        phase: "event_publish",
+                    });
+                    return err(error);
+                }
+                try {
+                    this.nats.publish(msg.reply, payload);
+                    await this.nats.flush();
+                }
+                catch (cause) {
+                    const error = new UnexpectedError({
+                        cause,
+                        context: { feed },
+                    });
+                    recordRuntimeError(error, {
+                        surface: "feed",
+                        direction: "server",
+                        operation: feed,
+                        phase: "event_publish",
+                    });
+                    return err(error);
                 }
-                this.nats.publish(msg.reply, payload);
-                await this.nats.flush();
                 return ok(undefined);
             })()),
         });
+        const handlerOutcome = isResultLike(handlerResult)
+            ? handlerResult.take()
+            : handlerResult;
+        if (isErr(handlerOutcome)) {
+            const error = annotateHandlerBoundaryError(handlerOutcome.error, {
+                feed,
+                requestId: msg.headers?.get("request-id"),
+                service: this.name,
+                contractId: this.contractId,
+                contractDigest: this.contractDigest,
+                traceId: traceIdFromTraceparent(msg.headers?.get("traceparent")),
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "server",
+                operation: feed,
+                phase: "handler_result",
+            });
+            return err(error);
+        }
         return ok(undefined);
     }
     finally {
@@ -1650,6 +1949,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     code: SpanStatusCode.ERROR,
                     message: "Failed to parse JSON",
                 });
+                recordRuntimeError(jsonData.error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "parse",
+                });
                 return jsonData;
             }
             const parsedInput = parseRuntimeSchema(ctx.input, jsonData).take();
@@ -1658,6 +1963,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     code: SpanStatusCode.ERROR,
                     message: "Input validation failed",
                 });
+                recordRuntimeError(parsedInput.error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "input_validation",
+                });
                 return parsedInput;
             }
             let caller;
@@ -1683,7 +1994,14 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         code: SpanStatusCode.ERROR,
                         message: "Missing session-key",
                     });
-                    return err(new AuthError({ reason: "missing_session_key" }));
+                    const error = new AuthError({ reason: "missing_session_key" });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 if (!proof) {
                     __classPrivateFieldGet(this, _Trellis_log, "f").warn({ method }, "Missing proof in request");
@@ -1691,11 +2009,25 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         code: SpanStatusCode.ERROR,
                         message: "Missing proof",
                     });
-                    return err(new AuthError({ reason: "missing_proof" }));
+                    const error = new AuthError({ reason: "missing_proof" });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 const iat = Number(iatHeader);
                 if (!Number.isSafeInteger(iat) || !requestId) {
-                    return err(new AuthError({ reason: "invalid_signature" }));
+                    const error = new AuthError({ reason: "invalid_signature" });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 // Verify proof signature locally using the raw request bytes we received.
                 const payloadBytes = msg.data ?? new Uint8Array();
@@ -1714,10 +2046,17 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         code: SpanStatusCode.ERROR,
                         message: "Invalid signature",
                     });
-                    return err(new AuthError({
+                    const error = new AuthError({
                         reason: "invalid_signature",
                         context: { sessionKey },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 let auth;
                 for (let attempt = 0; attempt < DEFAULT_AUTH_VALIDATE_SESSION_RETRY_ATTEMPTS; attempt++) {
@@ -1743,9 +2082,16 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     await sleep(DEFAULT_AUTH_VALIDATE_SESSION_RETRY_MS * (attempt + 1));
                 }
                 if (!auth) {
-                    return err(new UnexpectedError({
+                    const error = new UnexpectedError({
                         context: { reason: "missing_auth_validate_result" },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 if (auth instanceof Error) {
                     __classPrivateFieldGet(this, _Trellis_log, "f").warn({
@@ -1761,22 +2107,42 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         message: "Auth.Requests.Validate failed",
                     });
                     if (auth instanceof BaseError) {
+                        recordRuntimeError(auth, {
+                            surface: "rpc",
+                            direction: "server",
+                            operation: String(method),
+                            phase: "auth",
+                        });
                         return err(auth);
                     }
-                    return err(new UnexpectedError({ cause: auth }));
+                    const error = new UnexpectedError({ cause: auth });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 if (!auth.allowed) {
                     span.setStatus({
                         code: SpanStatusCode.ERROR,
                         message: "Insufficient permissions",
                     });
-                    return err(new AuthError({
+                    const error = new AuthError({
                         reason: "insufficient_permissions",
                         context: {
                             requiredCapabilities: ctx.callerCapabilities,
                             userCapabilities: auth.caller.capabilities,
                         },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 if (typeof msg.reply !== "string" ||
                     !msg.reply.startsWith(`${auth.inboxPrefix}.`)) {
@@ -1784,10 +2150,17 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         code: SpanStatusCode.ERROR,
                         message: "Reply subject mismatch",
                     });
-                    return err(new AuthError({
+                    const error = new AuthError({
                         reason: "reply_subject_mismatch",
                         context: { expected: auth.inboxPrefix, actual: msg.reply },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 caller = auth.caller;
             }
@@ -1815,7 +2188,14 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 client: handlerTrellis,
             })));
             if (handlerResultWrapped.isErr()) {
-                const error = handlerResultWrapped.error.withContext({ method });
+                const error = annotateHandlerBoundaryError(handlerResultWrapped.error, {
+                    method: String(method),
+                    requestId: msg.headers?.get("request-id"),
+                    service: this.name,
+                    contractId: this.contractId,
+                    contractDigest: this.contractDigest,
+                    traceId: activeTraceId(span) ?? incomingTraceId,
+                });
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({
                     method,
                     error: error.message,
@@ -1828,16 +2208,25 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     message: error.message,
                 });
                 span.recordException(error);
+                recordRuntimeError(error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "handler_throw",
+                });
                 return err(error);
             }
             const handlerResult = handlerResultWrapped.take();
             const handlerOutcome = handlerResult.take();
             if (isErr(handlerOutcome)) {
-                const handlerError = handlerOutcome.error;
-                const error = handlerError instanceof BaseError &&
-                    !(handlerError instanceof RemoteError)
-                    ? handlerError
-                    : new UnexpectedError({ cause: handlerError });
+                const error = annotateHandlerBoundaryError(handlerOutcome.error, {
+                    method: String(method),
+                    requestId: msg.headers?.get("request-id"),
+                    service: this.name,
+                    contractId: this.contractId,
+                    contractDigest: this.contractDigest,
+                    traceId: activeTraceId(span) ?? incomingTraceId,
+                });
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({
                     method,
                     error: error.message,
@@ -1850,6 +2239,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     code: SpanStatusCode.ERROR,
                     message: error.message,
                 });
+                recordRuntimeError(error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "handler_result",
+                });
                 return err(error);
             }
             const encoded = encodeSchema(ctx.output, handlerOutcome).take();
@@ -1858,6 +2253,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     code: SpanStatusCode.ERROR,
                     message: "Output encoding failed",
                 });
+                recordRuntimeError(encoded.error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "output_encoding",
+                });
                 return encoded;
             }
             span.setStatus({ code: SpanStatusCode.OK });
@@ -1934,17 +2335,31 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             const m = parsedEvent.take();
             if (isErr(m)) {
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({ error: m.error }, "Event validation failed");
+                recordRuntimeError(m.error, {
+                    surface: "event",
+                    direction: "consumer",
+                    operation: String(event),
+                    phase: "input_validation",
+                });
                 continue;
             }
-            const handlerResult = await AsyncResult.lift(fn(m, createEventListenerContext({
+            const handlerResult = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_invokeEventHandler).call(this, {
+                event,
                 payload: m,
-                subject: msg.subject,
                 mode: "ephemeral",
                 message: msg,
-            })));
-            if (handlerResult.isErr()) {
+                fn,
+            });
+            const handlerValue = handlerResult.take();
+            if (isErr(handlerValue)) {
+                recordRuntimeError(handlerValue.error, {
+                    surface: "event",
+                    direction: "consumer",
+                    operation: String(event),
+                    phase: "handler_result",
+                });
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({
-                    error: handlerResult.error.toSerializable(),
+                    error: handlerValue.error.toSerializable(),
                     event,
                     subject: msg.subject,
                 }, "Event handler failed");
@@ -1953,6 +2368,31 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
     });
     __classPrivateFieldGet(this, _Trellis_tasks, "f").add(`event:${event}:${ulid()}`, task);
     return ok(undefined);
+}, _Trellis_invokeEventHandler = async function _Trellis_invokeEventHandler(args) {
+    const annotation = {
+        event: String(args.event),
+        service: this.name,
+        contractId: this.contractId,
+        contractDigest: this.contractDigest,
+        traceId: traceIdFromTraceparent(args.message.headers?.get("traceparent")),
+    };
+    try {
+        const result = await Promise.resolve(args.fn(args.payload, createEventListenerContext({
+            payload: args.payload,
+            subject: args.message.subject,
+            mode: args.mode,
+            ...(args.group ? { group: args.group } : {}),
+            message: args.message,
+        })));
+        const outcome = isResultLike(result) ? result.take() : result;
+        if (isErr(outcome)) {
+            return err(annotateHandlerBoundaryError(outcome.error, annotation));
+        }
+        return ok(undefined);
+    }
+    catch (cause) {
+        return err(annotateHandlerBoundaryError(cause, annotation));
+    }
 }, _Trellis_resolveEventConsumerGroup = function _Trellis_resolveEventConsumerGroup(event, opts) {
     const metadata = __classPrivateFieldGet(this, _Trellis_eventConsumers, "f").metadata;
     const bindings = __classPrivateFieldGet(this, _Trellis_eventConsumers, "f").bindings ?? {};
@@ -2078,15 +2518,17 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 msg.term();
                 continue;
             }
-            const handlerResult = await AsyncResult.lift(fn(m, createEventListenerContext({
+            const handlerResult = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_invokeEventHandler).call(this, {
+                event,
                 payload: m,
-                subject: msg.subject,
                 mode: "durable",
                 message: msg,
-            })));
-            if (handlerResult.isErr()) {
+                fn,
+            });
+            const handlerValue = handlerResult.take();
+            if (isErr(handlerValue)) {
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({
-                    error: handlerResult.error.toSerializable(),
+                    error: handlerValue.error.toSerializable(),
                     event,
                     subject: msg.subject,
                 }, "Event handler failed");
@@ -2115,20 +2557,34 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 const eventPayload = parsedEvent.take();
                 if (isErr(eventPayload)) {
                     __classPrivateFieldGet(this, _Trellis_log, "f").error({ error: eventPayload.error }, "Event validation failed");
+                    recordRuntimeError(eventPayload.error, {
+                        surface: "event",
+                        direction: "consumer",
+                        operation: String(registration.event),
+                        phase: "input_validation",
+                    });
                     msg.term();
                     failed = true;
                     break;
                 }
-                const handlerResult = await AsyncResult.lift(registration.fn(eventPayload, createEventListenerContext({
+                const handlerResult = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_invokeEventHandler).call(this, {
+                    event: registration.event,
                     payload: eventPayload,
-                    subject: msg.subject,
                     mode: "durable",
                     group,
                     message: msg,
-                })));
-                if (handlerResult.isErr()) {
+                    fn: registration.fn,
+                });
+                const handlerValue = handlerResult.take();
+                if (isErr(handlerValue)) {
+                    recordRuntimeError(handlerValue.error, {
+                        surface: "event",
+                        direction: "consumer",
+                        operation: String(registration.event),
+                        phase: "handler_result",
+                    });
                     __classPrivateFieldGet(this, _Trellis_log, "f").error({
-                        error: handlerResult.error.toSerializable(),
+                        error: handlerValue.error.toSerializable(),
                         event: registration.event,
                         subject: msg.subject,
                     }, "Event handler failed");
@@ -2224,6 +2680,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         code: SpanStatusCode.ERROR,
                         message: response.error.message,
                     });
+                    recordRuntimeError(response.error, {
+                        surface: "operation",
+                        direction: "client",
+                        operation: "requestJson",
+                        phase: "request_send",
+                    });
                     return response;
                 }
                 const json = safeJson(response).take();
@@ -2239,6 +2701,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         code: SpanStatusCode.ERROR,
                         message: error.message,
                     });
+                    recordRuntimeError(error, {
+                        surface: "operation",
+                        direction: "client",
+                        operation: "requestJson",
+                        phase: "response_decoding",
+                    });
                     return err(error);
                 }
                 span.setStatus({ code: SpanStatusCode.OK });
@@ -2251,6 +2719,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     message: error.message,
                 });
                 span.recordException(error);
+                recordRuntimeError(error, {
+                    surface: "operation",
+                    direction: "client",
+                    operation: "requestJson",
+                    phase: "unexpected",
+                });
                 return err(error);
             }
             finally {
@@ -2278,35 +2752,56 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         }
         catch (cause) {
             sub.unsubscribe();
-            return err(createTransportError({
+            const error = createTransportError({
                 code: "trellis.watch.failed",
                 message: "Trellis could not start the operation watch.",
                 hint: "Retry watching the operation. If it keeps failing, reconnect to Trellis and try again.",
                 cause,
                 context: { subject },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "operation",
+                direction: "client",
+                operation: "watchJson",
+                phase: "request_send",
+            });
+            return err(error);
         }
         return ok((async function* () {
             try {
                 for await (const msg of sub) {
                     if (msg.headers?.get("status") === "error") {
-                        yield err(createTransportError({
+                        const error = createTransportError({
                             code: "trellis.watch.failed",
                             message: "Trellis stopped the operation watch.",
                             hint: "Retry watching the operation. If it keeps happening, reconnect to Trellis and try again.",
                             context: { subject, frame: msg.string() },
-                        }));
+                        });
+                        recordRuntimeError(error, {
+                            surface: "operation",
+                            direction: "client",
+                            operation: "watchJson",
+                            phase: "remote_error",
+                        });
+                        yield err(error);
                         continue;
                     }
                     const json = safeJson(msg).take();
                     if (isErr(json)) {
-                        yield err(createTransportError({
+                        const error = createTransportError({
                             code: "trellis.watch.invalid_response",
                             message: "Trellis returned an invalid watch update.",
                             hint: "Retry watching the operation. If it keeps happening, reconnect to Trellis and try again.",
                             cause: json.error.cause,
                             context: { subject },
-                        }));
+                        });
+                        recordRuntimeError(error, {
+                            surface: "operation",
+                            direction: "client",
+                            operation: "watchJson",
+                            phase: "response_decoding",
+                        });
+                        yield err(error);
                         continue;
                     }
                     yield ok(json);