@qlever-llc/trellis 0.10.10 → 0.10.12

package/src/trellis.ts

@@ -47,12 +47,14 @@ import {
   createNatsHeaderCarrier,
   extractTraceContext,
   injectTraceContext,
+  recordTrellisError,
   SpanStatusCode,
   startClientSpan,
   startServerSpan,
   trace,
+  type TrellisErrorMetricAttributes,
   withSpanAsync,
-} from "./tracing.js";
+} from "./telemetry/mod.js";
 import { Type } from "typebox";
 import { AssertError, Pointer } from "typebox/value";
 import { ulid } from "ulid";
@@ -732,6 +734,65 @@ export function isResultLike(
 ): value is Result<unknown, BaseError> {
   return value instanceof Result;
 }
+
+type SerializableRuntimeError = {
+  id?: string;
+  type: string;
+  message: string;
+  context?: Record<string, unknown>;
+  traceId?: string;
+} & Record<string, unknown>;
+
+export type HandlerErrorAnnotationContext = {
+  method?: string;
+  event?: string;
+  feed?: string;
+  operation?: string;
+  jobType?: string;
+  requestId?: string;
+  service?: string;
+  contractId?: string;
+  contractDigest?: string;
+  traceId?: string;
+};
+
+function compactHandlerErrorContext(
+  context: HandlerErrorAnnotationContext,
+): Record<string, unknown> {
+  return Object.fromEntries(
+    Object.entries(context).filter(([key, value]) =>
+      key !== "traceId" && value !== undefined
+    ),
+  );
+}
+
+function sanitizeHandlerErrorContext(error: BaseError): void {
+  delete error.getContext().subject;
+}
+
+export function annotateHandlerBoundaryError(
+  cause: unknown,
+  context: HandlerErrorAnnotationContext,
+): BaseError {
+  const error = cause instanceof BaseError && !(cause instanceof RemoteError)
+    ? cause
+    : new UnexpectedError({ cause });
+  sanitizeHandlerErrorContext(error);
+  error.withContext(compactHandlerErrorContext(context));
+  error.withTraceId(context.traceId);
+  return error;
+}
+
+function recordRuntimeError(
+  error: unknown,
+  attributes: TrellisErrorMetricAttributes,
+): void {
+  recordTrellisError(error, {
+    messagingSystem: "nats",
+    ...attributes,
+  });
+}
+
 export type RuntimeOperationDesc = {
   subject: string;
   input: unknown;
@@ -788,10 +849,7 @@ export type RuntimeOperationSnapshot = {
   progress?: unknown;
   transfer?: RuntimeOperationTransferProgress;
   output?: unknown;
-  error?: {
-    type: string;
-    message: string;
-  };
+  error?: SerializableRuntimeError;
 };
 
 export type RuntimeOperationRecord = {
@@ -848,8 +906,11 @@ const DurableOperationSnapshotSchema = Type.Object({
   })),
   output: Type.Optional(Type.Any()),
   error: Type.Optional(Type.Object({
+    id: Type.Optional(Type.String()),
     type: Type.String(),
     message: Type.String(),
+    context: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
+    traceId: Type.Optional(Type.String()),
   })),
 });
 
@@ -990,6 +1051,8 @@ export type TrellisOpts<TA extends AnyTrellisAPI> = {
   state?: RuntimeStateStores;
   connection?: TrellisConnection;
   onSessionNotFound?: () => MaybePromise<void>;
+  contractId?: string;
+  contractDigest?: string;
 };
 
 export type RequestOpts = {
@@ -1960,6 +2023,8 @@ export class Trellis<
   readonly handle: { readonly rpc: ActiveRpcHandleFacade<TA, TRequests> };
   /** Framework-neutral lifecycle handle for this Trellis runtime connection. */
   readonly connection: TrellisConnection;
+  readonly contractId?: string;
+  readonly contractDigest?: string;
 
   protected nats: NatsConnection;
   protected js: JetStreamClient;
@@ -1991,6 +2056,8 @@ export class Trellis<
     this.#log = (opts?.log ?? logger).child({ lib: "trellis" });
     this.timeout = opts?.timeout ?? 3000;
     this.stream = opts?.stream ?? "trellis";
+    this.contractId = opts?.contractId;
+    this.contractDigest = opts?.contractDigest;
     this.#hasExplicitApi = api !== undefined;
     this.#noResponderMaxRetries = opts?.noResponderRetry?.maxAttempts ??
       DEFAULT_NO_RESPONDER_MAX_RETRIES;
@@ -2409,11 +2476,23 @@ export class Trellis<
 
       const msg = encodeRuntimeSchema(ctx.input, input).take();
       if (isErr(msg)) {
+        recordRuntimeError(msg.error, {
+          surface: "rpc",
+          direction: "client",
+          operation: method,
+          phase: "request_encoding",
+        });
         return msg;
       }
 
       const subject = this.template(ctx.subject, input).take();
       if (isErr(subject)) {
+        recordRuntimeError(subject.error, {
+          surface: "rpc",
+          direction: "client",
+          operation: method,
+          phase: "request_encoding",
+        });
         return subject;
       }
 
@@ -2438,13 +2517,19 @@ export class Trellis<
         });
         const response = msgResult.take();
         if (isErr(response)) {
+          recordRuntimeError(response.error, {
+            surface: "rpc",
+            direction: "client",
+            operation: method,
+            phase: "request_send",
+          });
           return response;
         }
 
         if (response.headers?.get("status") === "error") {
           const json = safeJson(response).take();
           if (isErr(json)) {
-            return err(requestFailedTransportError({
+            const error = requestFailedTransportError({
               code: "trellis.request.invalid_response",
               message: "Trellis returned an invalid response.",
               hint:
@@ -2452,12 +2537,19 @@ export class Trellis<
               method,
               subject,
               cause: json.error.cause,
-            }));
+            });
+            recordRuntimeError(error, {
+              surface: "rpc",
+              direction: "client",
+              operation: method,
+              phase: "response_decoding",
+            });
+            return err(error);
           }
 
           const errorData = parse(TrellisErrorDataSchema, json).take();
           if (isErr(errorData)) {
-            return err(requestFailedTransportError({
+            const error = requestFailedTransportError({
               code: "trellis.request.invalid_response",
               message: "Trellis returned an invalid response.",
               hint:
@@ -2465,7 +2557,14 @@ export class Trellis<
               method,
               subject,
               cause: errorData.error,
-            }));
+            });
+            recordRuntimeError(error, {
+              surface: "rpc",
+              direction: "client",
+              operation: method,
+              phase: "response_decoding",
+            });
+            return err(error);
           }
 
           const declaredErrorTypes = Array.isArray(ctx.declaredErrorTypes)
@@ -2484,17 +2583,29 @@ export class Trellis<
           );
           if (reconstructed) {
             await this.#handleBrowserAuthRequired(reconstructed);
+            recordRuntimeError(new RemoteError({ error: errorData }), {
+              surface: "rpc",
+              direction: "client",
+              operation: method,
+              phase: "remote_error",
+            });
             return err(reconstructed);
           }
 
           const remoteError = new RemoteError({ error: errorData });
           await this.#handleBrowserAuthRequired(remoteError);
+          recordRuntimeError(remoteError, {
+            surface: "rpc",
+            direction: "client",
+            operation: method,
+            phase: "remote_error",
+          });
           return err(remoteError);
         }
 
         const json = safeJson(response).take();
         if (isErr(json)) {
-          return err(requestFailedTransportError({
+          const error = requestFailedTransportError({
             code: "trellis.request.invalid_response",
             message: "Trellis returned an invalid response.",
             hint:
@@ -2502,11 +2613,24 @@ export class Trellis<
             method,
             subject,
             cause: json.error.cause,
-          }));
+          });
+          recordRuntimeError(error, {
+            surface: "rpc",
+            direction: "client",
+            operation: method,
+            phase: "response_decoding",
+          });
+          return err(error);
         }
 
         const outputResult = parseRuntimeSchema(ctx.output, json).take();
         if (isErr(outputResult)) {
+          recordRuntimeError(outputResult.error, {
+            surface: "rpc",
+            direction: "client",
+            operation: method,
+            phase: "response_decoding",
+          });
           return err(outputResult.error);
         }
 
@@ -2535,6 +2659,12 @@ export class Trellis<
             message: unexpected.message,
           });
           span.recordException(unexpected);
+          recordRuntimeError(unexpected, {
+            surface: "rpc",
+            direction: "client",
+            operation: method,
+            phase: "unexpected",
+          });
           return err(unexpected);
         } finally {
           span.end();
@@ -2683,13 +2813,29 @@ export class Trellis<
   ): AsyncResult<FeedSubscription<TEvent>, BaseError> {
     return AsyncResult.from((async () => {
       const payload = encodeRuntimeSchema(descriptor.input, input).take();
-      if (isErr(payload)) return payload;
+      if (isErr(payload)) {
+        recordRuntimeError(payload.error, {
+          surface: "feed",
+          direction: "client",
+          operation: feed,
+          phase: "request_encoding",
+        });
+        return payload;
+      }
 
       const subject = this.template(
         descriptor.subject,
         input as Record<string, unknown>,
       ).take();
-      if (isErr(subject)) return subject;
+      if (isErr(subject)) {
+        recordRuntimeError(subject.error, {
+          surface: "feed",
+          direction: "client",
+          operation: feed,
+          phase: "request_template",
+        });
+        return subject;
+      }
 
       const authHeaders = await this.#createProof(subject, payload);
       const headers = natsHeaders();
@@ -2711,14 +2857,21 @@ export class Trellis<
       } catch (cause) {
         opts?.signal?.removeEventListener("abort", abort);
         sub.unsubscribe();
-        return err(createTransportError({
+        const error = createTransportError({
           code: "trellis.feed.subscribe_failed",
           message: "Trellis could not subscribe to the feed.",
           hint:
             "Retry the subscription. If it keeps failing, check Trellis runtime health.",
           cause,
           context: { feed, subject },
-        }));
+        });
+        recordRuntimeError(error, {
+          surface: "feed",
+          direction: "client",
+          operation: feed,
+          phase: "request_send",
+        });
+        return err(error);
       }
 
       let timeoutId: ReturnType<typeof setTimeout> | undefined;
@@ -2749,7 +2902,7 @@ export class Trellis<
       if (firstFrame === "timeout" || firstFrame === "aborted") {
         opts?.signal?.removeEventListener("abort", abort);
         sub.unsubscribe();
-        return err(createTransportError({
+        const error = createTransportError({
           code: firstFrame === "timeout"
             ? "trellis.feed.subscribe_timeout"
             : "trellis.feed.subscribe_aborted",
@@ -2760,30 +2913,51 @@ export class Trellis<
             ? "Check that the target service is running and has the current deployment digest, then retry."
             : "Retry the subscription if the feed is still needed.",
           context: { feed, subject },
-        }));
+        });
+        recordRuntimeError(error, {
+          surface: "feed",
+          direction: "client",
+          operation: feed,
+          phase: "handshake",
+        });
+        return err(error);
       }
       if (firstFrame.done) {
         opts?.signal?.removeEventListener("abort", abort);
         sub.unsubscribe();
-        return err(createTransportError({
+        const error = createTransportError({
           code: "trellis.feed.subscribe_closed",
           message: "Trellis closed the feed before acknowledging it.",
           hint:
             "Retry the subscription. If it keeps failing, check Trellis runtime health.",
           context: { feed, subject },
-        }));
+        });
+        recordRuntimeError(error, {
+          surface: "feed",
+          direction: "client",
+          operation: feed,
+          phase: "handshake",
+        });
+        return err(error);
       }
       const firstMessage = firstFrame.value;
       if (firstMessage.headers?.get("status") === "error") {
         opts?.signal?.removeEventListener("abort", abort);
         sub.unsubscribe();
-        return err(createTransportError({
+        const error = createTransportError({
           code: "trellis.feed.failed",
           message: "Trellis rejected the feed subscription.",
           hint:
             "Retry the subscription. If it keeps failing, check Trellis runtime health and permissions.",
           context: { feed, subject, frame: firstMessage.string() },
-        }));
+        });
+        recordRuntimeError(error, {
+          surface: "feed",
+          direction: "client",
+          operation: feed,
+          phase: "remote_error",
+        });
+        return err(error);
       }
       const firstEvent = firstMessage.headers?.get("feed-status") === "ready"
         ? undefined
@@ -2794,18 +2968,41 @@ export class Trellis<
         try {
           const parseFeedFrame = (msg: Msg): TEvent => {
             if (msg.headers?.get("status") === "error") {
-              throw createTransportError({
+              const error = createTransportError({
                 code: "trellis.feed.failed",
                 message: "Trellis stopped the feed.",
                 hint:
                   "Retry the subscription. If it keeps failing, check Trellis runtime health.",
                 context: { feed, subject, frame: msg.string() },
               });
+              recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "remote_error",
+              });
+              throw error;
             }
             const json = safeJson(msg).take();
-            if (isErr(json)) throw json.error;
+            if (isErr(json)) {
+              recordRuntimeError(json.error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "event_decoding",
+              });
+              throw json.error;
+            }
             const parsed = parseRuntimeSchema(eventSchema, json).take();
-            if (isErr(parsed)) throw parsed.error;
+            if (isErr(parsed)) {
+              recordRuntimeError(parsed.error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "event_validation",
+              });
+              throw parsed.error;
+            }
             return parsed as TEvent;
           };
           if (firstEvent) yield parseFeedFrame(firstEvent);
@@ -2836,7 +3033,7 @@ export class Trellis<
       sub = this.nats.subscribe(subject);
       await this.nats.flush();
     } catch (cause) {
-      throw createTransportError({
+      const error = createTransportError({
         code: "trellis.feed.listen_failed",
         message: "Trellis could not listen for feed requests.",
         hint:
@@ -2844,6 +3041,13 @@ export class Trellis<
         cause,
         context: { feed, subject },
       });
+      recordRuntimeError(error, {
+        surface: "feed",
+        direction: "server",
+        operation: feed,
+        phase: "listen",
+      });
+      throw error;
     }
     const task = AsyncResult.try(async () => {
       for await (const msg of sub) {
@@ -2860,9 +3064,20 @@ export class Trellis<
               this.#respondWithError(msg, value.error);
             }
           } catch (cause) {
-            const error = cause instanceof BaseError
-              ? cause
-              : new UnexpectedError({ cause });
+            const error = annotateHandlerBoundaryError(cause, {
+              feed,
+              requestId: msg.headers?.get("request-id"),
+              service: this.name,
+              contractId: this.contractId,
+              contractDigest: this.contractDigest,
+              traceId: traceIdFromTraceparent(msg.headers?.get("traceparent")),
+            });
+            recordRuntimeError(error, {
+              surface: "feed",
+              direction: "server",
+              operation: feed,
+              phase: "handler_throw",
+            });
             this.#respondWithError(msg, error);
           }
         })();
@@ -2880,9 +3095,25 @@ export class Trellis<
     ) => unknown | Promise<unknown>,
   ): Promise<Result<void, BaseError>> {
     const json = safeJson(msg).take();
-    if (isErr(json)) return json;
+    if (isErr(json)) {
+      recordRuntimeError(json.error, {
+        surface: "feed",
+        direction: "server",
+        operation: feed,
+        phase: "request_decoding",
+      });
+      return json;
+    }
     const parsed = parseRuntimeSchema(descriptor.input, json).take();
-    if (isErr(parsed)) return parsed;
+    if (isErr(parsed)) {
+      recordRuntimeError(parsed.error, {
+        surface: "feed",
+        direction: "server",
+        operation: feed,
+        phase: "input_validation",
+      });
+      return parsed;
+    }
 
     const caller = await this.#authenticateFeedRequest({
       feed,
@@ -2892,13 +3123,26 @@ export class Trellis<
       requiredCapabilities: descriptor.subscribeCapabilities,
     });
     const callerValue = caller.take();
-    if (isErr(callerValue)) return callerValue;
+    if (isErr(callerValue)) {
+      recordRuntimeError(callerValue.error, {
+        surface: "feed",
+        direction: "server",
+        operation: feed,
+        phase: "auth",
+      });
+      return callerValue;
+    }
     if (!msg.reply) {
-      return err(
-        new UnexpectedError({
-          context: { feed, reason: "missing_reply" },
-        }),
-      );
+      const error = new UnexpectedError({
+        context: { feed, reason: "missing_reply" },
+      });
+      recordRuntimeError(error, {
+        surface: "feed",
+        direction: "server",
+        operation: feed,
+        phase: "handshake",
+      });
+      return err(error);
     }
     const readyHeaders = natsHeaders();
     readyHeaders.set("feed-status", "ready");
@@ -2907,26 +3151,73 @@ export class Trellis<
 
     const controller = new AbortController();
     try {
-      await handler({
+      const handlerResult = await handler({
         input: parsed as TInput,
         caller: callerValue,
         signal: controller.signal,
         emit: (event: TEvent) =>
           AsyncResult.from((async () => {
             const payload = encodeRuntimeSchema(descriptor.event, event).take();
-            if (isErr(payload)) return payload;
+            if (isErr(payload)) {
+              recordRuntimeError(payload.error, {
+                surface: "feed",
+                direction: "server",
+                operation: feed,
+                phase: "event_encoding",
+              });
+              return payload;
+            }
             if (!msg.reply) {
-              return err(
-                new UnexpectedError({
-                  context: { feed, reason: "missing_reply" },
-                }),
-              );
+              const error = new UnexpectedError({
+                context: { feed, reason: "missing_reply" },
+              });
+              recordRuntimeError(error, {
+                surface: "feed",
+                direction: "server",
+                operation: feed,
+                phase: "event_publish",
+              });
+              return err(error);
+            }
+            try {
+              this.nats.publish(msg.reply, payload);
+              await this.nats.flush();
+            } catch (cause) {
+              const error = new UnexpectedError({
+                cause,
+                context: { feed },
+              });
+              recordRuntimeError(error, {
+                surface: "feed",
+                direction: "server",
+                operation: feed,
+                phase: "event_publish",
+              });
+              return err(error);
             }
-            this.nats.publish(msg.reply, payload);
-            await this.nats.flush();
             return ok(undefined);
           })()),
       });
+      const handlerOutcome = isResultLike(handlerResult)
+        ? handlerResult.take()
+        : handlerResult;
+      if (isErr(handlerOutcome)) {
+        const error = annotateHandlerBoundaryError(handlerOutcome.error, {
+          feed,
+          requestId: msg.headers?.get("request-id"),
+          service: this.name,
+          contractId: this.contractId,
+          contractDigest: this.contractDigest,
+          traceId: traceIdFromTraceparent(msg.headers?.get("traceparent")),
+        });
+        recordRuntimeError(error, {
+          surface: "feed",
+          direction: "server",
+          operation: feed,
+          phase: "handler_result",
+        });
+        return err(error);
+      }
       return ok(undefined);
     } finally {
       controller.abort();
@@ -3113,6 +3404,12 @@ export class Trellis<
             code: SpanStatusCode.ERROR,
             message: "Failed to parse JSON",
           });
+          recordRuntimeError(jsonData.error, {
+            surface: "rpc",
+            direction: "server",
+            operation: String(method),
+            phase: "parse",
+          });
           return jsonData;
         }
 
@@ -3122,6 +3419,12 @@ export class Trellis<
             code: SpanStatusCode.ERROR,
             message: "Input validation failed",
           });
+          recordRuntimeError(parsedInput.error, {
+            surface: "rpc",
+            direction: "server",
+            operation: String(method),
+            phase: "input_validation",
+          });
           return parsedInput;
         }
 
@@ -3148,7 +3451,14 @@ export class Trellis<
               code: SpanStatusCode.ERROR,
               message: "Missing session-key",
             });
-            return err(new AuthError({ reason: "missing_session_key" }));
+            const error = new AuthError({ reason: "missing_session_key" });
+            recordRuntimeError(error, {
+              surface: "rpc",
+              direction: "server",
+              operation: String(method),
+              phase: "auth",
+            });
+            return err(error);
           }
           if (!proof) {
             this.#log.warn({ method }, "Missing proof in request");
@@ -3156,11 +3466,25 @@ export class Trellis<
               code: SpanStatusCode.ERROR,
               message: "Missing proof",
             });
-            return err(new AuthError({ reason: "missing_proof" }));
+            const error = new AuthError({ reason: "missing_proof" });
+            recordRuntimeError(error, {
+              surface: "rpc",
+              direction: "server",
+              operation: String(method),
+              phase: "auth",
+            });
+            return err(error);
           }
           const iat = Number(iatHeader);
           if (!Number.isSafeInteger(iat) || !requestId) {
-            return err(new AuthError({ reason: "invalid_signature" }));
+            const error = new AuthError({ reason: "invalid_signature" });
+            recordRuntimeError(error, {
+              surface: "rpc",
+              direction: "server",
+              operation: String(method),
+              phase: "auth",
+            });
+            return err(error);
           }
 
           // Verify proof signature locally using the raw request bytes we received.
@@ -3199,12 +3523,17 @@ export class Trellis<
               code: SpanStatusCode.ERROR,
               message: "Invalid signature",
             });
-            return err(
-              new AuthError({
-                reason: "invalid_signature",
-                context: { sessionKey },
-              }),
-            );
+            const error = new AuthError({
+              reason: "invalid_signature",
+              context: { sessionKey },
+            });
+            recordRuntimeError(error, {
+              surface: "rpc",
+              direction: "server",
+              operation: String(method),
+              phase: "auth",
+            });
+            return err(error);
           }
 
           let auth:
@@ -3250,11 +3579,16 @@ export class Trellis<
           }
 
           if (!auth) {
-            return err(
-              new UnexpectedError({
-                context: { reason: "missing_auth_validate_result" },
-              }),
-            );
+            const error = new UnexpectedError({
+              context: { reason: "missing_auth_validate_result" },
+            });
+            recordRuntimeError(error, {
+              surface: "rpc",
+              direction: "server",
+              operation: String(method),
+              phase: "auth",
+            });
+            return err(error);
           }
 
           if (auth instanceof Error) {
@@ -3274,9 +3608,22 @@ export class Trellis<
               message: "Auth.Requests.Validate failed",
             });
             if (auth instanceof BaseError) {
+              recordRuntimeError(auth, {
+                surface: "rpc",
+                direction: "server",
+                operation: String(method),
+                phase: "auth",
+              });
               return err(auth);
             }
-            return err(new UnexpectedError({ cause: auth }));
+            const error = new UnexpectedError({ cause: auth });
+            recordRuntimeError(error, {
+              surface: "rpc",
+              direction: "server",
+              operation: String(method),
+              phase: "auth",
+            });
+            return err(error);
           }
 
           if (!auth.allowed) {
@@ -3284,15 +3631,20 @@ export class Trellis<
               code: SpanStatusCode.ERROR,
               message: "Insufficient permissions",
             });
-            return err(
-              new AuthError({
-                reason: "insufficient_permissions",
-                context: {
-                  requiredCapabilities: ctx.callerCapabilities,
-                  userCapabilities: auth.caller.capabilities,
-                },
-              }),
-            );
+            const error = new AuthError({
+              reason: "insufficient_permissions",
+              context: {
+                requiredCapabilities: ctx.callerCapabilities,
+                userCapabilities: auth.caller.capabilities,
+              },
+            });
+            recordRuntimeError(error, {
+              surface: "rpc",
+              direction: "server",
+              operation: String(method),
+              phase: "auth",
+            });
+            return err(error);
           }
 
           if (
@@ -3303,12 +3655,17 @@ export class Trellis<
               code: SpanStatusCode.ERROR,
               message: "Reply subject mismatch",
             });
-            return err(
-              new AuthError({
-                reason: "reply_subject_mismatch",
-                context: { expected: auth.inboxPrefix, actual: msg.reply },
-              }),
-            );
+            const error = new AuthError({
+              reason: "reply_subject_mismatch",
+              context: { expected: auth.inboxPrefix, actual: msg.reply },
+            });
+            recordRuntimeError(error, {
+              surface: "rpc",
+              direction: "server",
+              operation: String(method),
+              phase: "auth",
+            });
+            return err(error);
           }
 
           caller = auth.caller;
@@ -3350,7 +3707,17 @@ export class Trellis<
         );
 
         if (handlerResultWrapped.isErr()) {
-          const error = handlerResultWrapped.error.withContext({ method });
+          const error = annotateHandlerBoundaryError(
+            handlerResultWrapped.error,
+            {
+              method: String(method),
+              requestId: msg.headers?.get("request-id"),
+              service: this.name,
+              contractId: this.contractId,
+              contractDigest: this.contractDigest,
+              traceId: activeTraceId(span) ?? incomingTraceId,
+            },
+          );
           this.#log.error(
             {
               method,
@@ -3366,6 +3733,12 @@ export class Trellis<
             message: error.message,
           });
           span.recordException(error);
+          recordRuntimeError(error, {
+            surface: "rpc",
+            direction: "server",
+            operation: String(method),
+            phase: "handler_throw",
+          });
           return err(error);
         }
 
@@ -3374,12 +3747,14 @@ export class Trellis<
         };
         const handlerOutcome = handlerResult.take();
         if (isErr(handlerOutcome)) {
-          const handlerError = handlerOutcome.error;
-
-          const error = handlerError instanceof BaseError &&
-              !(handlerError instanceof RemoteError)
-            ? handlerError
-            : new UnexpectedError({ cause: handlerError });
+          const error = annotateHandlerBoundaryError(handlerOutcome.error, {
+            method: String(method),
+            requestId: msg.headers?.get("request-id"),
+            service: this.name,
+            contractId: this.contractId,
+            contractDigest: this.contractDigest,
+            traceId: activeTraceId(span) ?? incomingTraceId,
+          });
 
           this.#log.error(
             {
@@ -3396,6 +3771,12 @@ export class Trellis<
             code: SpanStatusCode.ERROR,
             message: error.message,
           });
+          recordRuntimeError(error, {
+            surface: "rpc",
+            direction: "server",
+            operation: String(method),
+            phase: "handler_result",
+          });
           return err(error);
         }
 
@@ -3405,6 +3786,12 @@ export class Trellis<
             code: SpanStatusCode.ERROR,
             message: "Output encoding failed",
           });
+          recordRuntimeError(encoded.error, {
+            surface: "rpc",
+            direction: "server",
+            operation: String(method),
+            phase: "output_encoding",
+          });
           return encoded;
         }
 
@@ -3529,17 +3916,28 @@ export class Trellis<
         typeof eventName
       >;
       if (!ctx) {
-        return err(
-          new UnexpectedError({
-            cause: this.#unknownApiError("event", event.toString()),
-            context: { event: event.toString() },
-          }),
-        );
+        const error = new UnexpectedError({
+          cause: this.#unknownApiError("event", event.toString()),
+          context: { event: event.toString() },
+        });
+        recordRuntimeError(error, {
+          surface: "event",
+          direction: "publisher",
+          operation: event,
+          phase: "prepare",
+        });
+        return err(error);
       }
 
       const subject = this.template(ctx.subject, data).take();
       if (isErr(subject)) {
         logger.error({ err: subject.error }, "Failed to template event.");
+        recordRuntimeError(subject.error, {
+          surface: "event",
+          direction: "publisher",
+          operation: event,
+          phase: "request_encoding",
+        });
         return subject;
       }
 
@@ -3554,7 +3952,14 @@ export class Trellis<
       const msg = encodeSchema(ctx.event, payload).take();
       if (isErr(msg)) {
         logger.error({ err: msg.error }, "Failed to encode event.");
-        return err(new UnexpectedError({ cause: msg.error }));
+        const error = new UnexpectedError({ cause: msg.error });
+        recordRuntimeError(error, {
+          surface: "event",
+          direction: "publisher",
+          operation: event,
+          phase: "request_encoding",
+        });
+        return err(error);
       }
 
       const headers = natsHeaders();
@@ -3574,9 +3979,17 @@ export class Trellis<
         headers: Object.freeze(headerRecord),
       }));
     } catch (cause) {
-      return err(
-        new UnexpectedError({ cause, context: { event: event.toString() } }),
-      );
+      const error = new UnexpectedError({
+        cause,
+        context: { event: event.toString() },
+      });
+      recordRuntimeError(error, {
+        surface: "event",
+        direction: "publisher",
+        operation: event,
+        phase: "prepare",
+      });
+      return err(error);
     }
   }
 
@@ -3601,9 +4014,17 @@ export class Trellis<
         await this.js.publish(event.subject, event.encodedPayload, { headers });
         return ok(undefined);
       } catch (cause) {
-        return err(
-          new UnexpectedError({ cause, context: { event: event.event } }),
-        );
+        const error = new UnexpectedError({
+          cause,
+          context: { event: event.event },
+        });
+        recordRuntimeError(error, {
+          surface: "event",
+          direction: "publisher",
+          operation: event.event,
+          phase: "publish",
+        });
+        return err(error);
       }
     })());
   }
@@ -3710,24 +4131,33 @@ export class Trellis<
         const m = parsedEvent.take();
         if (isErr(m)) {
           this.#log.error({ error: m.error }, "Event validation failed");
+          recordRuntimeError(m.error, {
+            surface: "event",
+            direction: "consumer",
+            operation: String(event),
+            phase: "input_validation",
+          });
           continue;
         }
 
-        const handlerResult = await AsyncResult.lift(
-          fn(
-            m as EventOf<TA, EventsOf<TA>>,
-            createEventListenerContext({
-              payload: m,
-              subject: msg.subject,
-              mode: "ephemeral",
-              message: msg,
-            }),
-          ),
-        );
-        if (handlerResult.isErr()) {
+        const handlerResult = await this.#invokeEventHandler({
+          event,
+          payload: m,
+          mode: "ephemeral",
+          message: msg,
+          fn,
+        });
+        const handlerValue = handlerResult.take();
+        if (isErr(handlerValue)) {
+          recordRuntimeError(handlerValue.error, {
+            surface: "event",
+            direction: "consumer",
+            operation: String(event),
+            phase: "handler_result",
+          });
           this.#log.error(
             {
-              error: handlerResult.error.toSerializable(),
+              error: handlerValue.error.toSerializable(),
               event,
               subject: msg.subject,
             },
@@ -3741,6 +4171,42 @@ export class Trellis<
     return ok(undefined);
   }
 
+  async #invokeEventHandler(args: {
+    event: EventsOf<TA>;
+    payload: unknown;
+    mode: "durable" | "ephemeral";
+    group?: string;
+    message: Pick<Msg, "headers" | "subject"> & object;
+    fn: EventCallback<EventOf<TA, EventsOf<TA>>>;
+  }): Promise<Result<void, BaseError>> {
+    const annotation = {
+      event: String(args.event),
+      service: this.name,
+      contractId: this.contractId,
+      contractDigest: this.contractDigest,
+      traceId: traceIdFromTraceparent(args.message.headers?.get("traceparent")),
+    };
+    try {
+      const result = await Promise.resolve(args.fn(
+        args.payload as EventOf<TA, EventsOf<TA>>,
+        createEventListenerContext({
+          payload: args.payload,
+          subject: args.message.subject,
+          mode: args.mode,
+          ...(args.group ? { group: args.group } : {}),
+          message: args.message,
+        }),
+      ));
+      const outcome = isResultLike(result) ? result.take() : result;
+      if (isErr(outcome)) {
+        return err(annotateHandlerBoundaryError(outcome.error, annotation));
+      }
+      return ok(undefined);
+    } catch (cause) {
+      return err(annotateHandlerBoundaryError(cause, annotation));
+    }
+  }
+
   #resolveEventConsumerGroup(
     event: EventsOf<TA>,
     opts: EventOpts | undefined,
@@ -3944,21 +4410,18 @@ export class Trellis<
           continue;
         }
 
-        const handlerResult = await AsyncResult.lift(
-          fn(
-            m as EventOf<TA, EventsOf<TA>>,
-            createEventListenerContext({
-              payload: m,
-              subject: msg.subject,
-              mode: "durable",
-              message: msg,
-            }),
-          ),
-        );
-        if (handlerResult.isErr()) {
+        const handlerResult = await this.#invokeEventHandler({
+          event,
+          payload: m,
+          mode: "durable",
+          message: msg,
+          fn,
+        });
+        const handlerValue = handlerResult.take();
+        if (isErr(handlerValue)) {
           this.#log.error(
             {
-              error: handlerResult.error.toSerializable(),
+              error: handlerValue.error.toSerializable(),
               event,
               subject: msg.subject,
             },
@@ -4009,27 +4472,36 @@ export class Trellis<
               { error: eventPayload.error },
               "Event validation failed",
             );
+            recordRuntimeError(eventPayload.error, {
+              surface: "event",
+              direction: "consumer",
+              operation: String(registration.event),
+              phase: "input_validation",
+            });
             msg.term();
             failed = true;
             break;
           }
 
-          const handlerResult = await AsyncResult.lift(
-            registration.fn(
-              eventPayload as EventOf<TA, EventsOf<TA>>,
-              createEventListenerContext({
-                payload: eventPayload,
-                subject: msg.subject,
-                mode: "durable",
-                group,
-                message: msg,
-              }),
-            ),
-          );
-          if (handlerResult.isErr()) {
+          const handlerResult = await this.#invokeEventHandler({
+            event: registration.event,
+            payload: eventPayload,
+            mode: "durable",
+            group,
+            message: msg,
+            fn: registration.fn,
+          });
+          const handlerValue = handlerResult.take();
+          if (isErr(handlerValue)) {
+            recordRuntimeError(handlerValue.error, {
+              surface: "event",
+              direction: "consumer",
+              operation: String(registration.event),
+              phase: "handler_result",
+            });
             this.#log.error(
               {
-                error: handlerResult.error.toSerializable(),
+                error: handlerValue.error.toSerializable(),
                 event: registration.event,
                 subject: msg.subject,
               },
@@ -4236,6 +4708,12 @@ export class Trellis<
               code: SpanStatusCode.ERROR,
               message: response.error.message,
             });
+            recordRuntimeError(response.error, {
+              surface: "operation",
+              direction: "client",
+              operation: "requestJson",
+              phase: "request_send",
+            });
             return response;
           }
 
@@ -4253,6 +4731,12 @@ export class Trellis<
               code: SpanStatusCode.ERROR,
               message: error.message,
             });
+            recordRuntimeError(error, {
+              surface: "operation",
+              direction: "client",
+              operation: "requestJson",
+              phase: "response_decoding",
+            });
             return err(error);
           }
 
@@ -4265,6 +4749,12 @@ export class Trellis<
             message: error.message,
           });
           span.recordException(error);
+          recordRuntimeError(error, {
+            surface: "operation",
+            direction: "client",
+            operation: "requestJson",
+            phase: "unexpected",
+          });
           return err(error);
         } finally {
           span.end();
@@ -4301,40 +4791,61 @@ export class Trellis<
         await this.nats.flush();
       } catch (cause) {
         sub.unsubscribe();
-        return err(createTransportError({
+        const error = createTransportError({
           code: "trellis.watch.failed",
           message: "Trellis could not start the operation watch.",
           hint:
             "Retry watching the operation. If it keeps failing, reconnect to Trellis and try again.",
           cause,
           context: { subject },
-        }));
+        });
+        recordRuntimeError(error, {
+          surface: "operation",
+          direction: "client",
+          operation: "watchJson",
+          phase: "request_send",
+        });
+        return err(error);
       }
 
       return ok((async function* () {
         try {
           for await (const msg of sub) {
             if (msg.headers?.get("status") === "error") {
-              yield err(createTransportError({
+              const error = createTransportError({
                 code: "trellis.watch.failed",
                 message: "Trellis stopped the operation watch.",
                 hint:
                   "Retry watching the operation. If it keeps happening, reconnect to Trellis and try again.",
                 context: { subject, frame: msg.string() },
-              }));
+              });
+              recordRuntimeError(error, {
+                surface: "operation",
+                direction: "client",
+                operation: "watchJson",
+                phase: "remote_error",
+              });
+              yield err(error);
               continue;
             }
 
             const json = safeJson(msg).take();
             if (isErr(json)) {
-              yield err(createTransportError({
+              const error = createTransportError({
                 code: "trellis.watch.invalid_response",
                 message: "Trellis returned an invalid watch update.",
                 hint:
                   "Retry watching the operation. If it keeps happening, reconnect to Trellis and try again.",
                 cause: json.error.cause,
                 context: { subject },
-              }));
+              });
+              recordRuntimeError(error, {
+                surface: "operation",
+                direction: "client",
+                operation: "watchJson",
+                phase: "response_decoding",
+              });
+              yield err(error);
               continue;
             }