@qlever-llc/trellis 0.10.11 → 0.10.12

package/esm/trellis.js

@@ -14,7 +14,7 @@ import { jetstream, jetstreamManager, } from "@nats-io/jetstream";
 import { createInbox, headers as natsHeaders, } from "@nats-io/nats-core";
 import { CONTRACT_JOBS_METADATA, CONTRACT_KV_METADATA, CONTRACT_STATE_METADATA, } from "./contract_support/mod.js";
 import { AsyncResult, BaseError, err, isErr, ok, Result, } from "@qlever-llc/result";
-import { createNatsHeaderCarrier, extractTraceContext, injectTraceContext, SpanStatusCode, startClientSpan, startServerSpan, withSpanAsync, } from "./tracing.js";
+import { createNatsHeaderCarrier, extractTraceContext, injectTraceContext, recordTrellisError, SpanStatusCode, startClientSpan, startServerSpan, withSpanAsync, } from "./telemetry/mod.js";
 import { Type } from "typebox";
 import { Pointer } from "typebox/value";
 import { ulid } from "ulid";
@@ -205,6 +205,12 @@ export function annotateHandlerBoundaryError(cause, context) {
     error.withTraceId(context.traceId);
     return error;
 }
+function recordRuntimeError(error, attributes) {
+    recordTrellisError(error, {
+        messagingSystem: "nats",
+        ...attributes,
+    });
+}
 const DurableOperationSignalSchema = Type.Object({
     operationId: Type.String(),
     sequence: Type.Number(),
@@ -935,14 +941,27 @@ export class Trellis {
             const eventName = event;
             const ctx = this.api["events"][eventName];
             if (!ctx) {
-                return err(new UnexpectedError({
+                const error = new UnexpectedError({
                     cause: __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_unknownApiError).call(this, "event", event.toString()),
                     context: { event: event.toString() },
-                }));
+                });
+                recordRuntimeError(error, {
+                    surface: "event",
+                    direction: "publisher",
+                    operation: event,
+                    phase: "prepare",
+                });
+                return err(error);
             }
             const subject = this.template(ctx.subject, data).take();
             if (isErr(subject)) {
                 logger.error({ err: subject.error }, "Failed to template event.");
+                recordRuntimeError(subject.error, {
+                    surface: "event",
+                    direction: "publisher",
+                    operation: event,
+                    phase: "request_encoding",
+                });
                 return subject;
             }
             const header = {
@@ -956,7 +975,14 @@ export class Trellis {
             const msg = encodeSchema(ctx.event, payload).take();
             if (isErr(msg)) {
                 logger.error({ err: msg.error }, "Failed to encode event.");
-                return err(new UnexpectedError({ cause: msg.error }));
+                const error = new UnexpectedError({ cause: msg.error });
+                recordRuntimeError(error, {
+                    surface: "event",
+                    direction: "publisher",
+                    operation: event,
+                    phase: "request_encoding",
+                });
+                return err(error);
             }
             const headers = natsHeaders();
             headers.set("Nats-Msg-Id", header.id);
@@ -974,7 +1000,17 @@ export class Trellis {
             }));
         }
         catch (cause) {
-            return err(new UnexpectedError({ cause, context: { event: event.toString() } }));
+            const error = new UnexpectedError({
+                cause,
+                context: { event: event.toString() },
+            });
+            recordRuntimeError(error, {
+                surface: "event",
+                direction: "publisher",
+                operation: event,
+                phase: "prepare",
+            });
+            return err(error);
         }
     }
     /**
@@ -993,7 +1029,17 @@ export class Trellis {
                 return ok(undefined);
             }
             catch (cause) {
-                return err(new UnexpectedError({ cause, context: { event: event.event } }));
+                const error = new UnexpectedError({
+                    cause,
+                    context: { event: event.event },
+                });
+                recordRuntimeError(error, {
+                    surface: "event",
+                    direction: "publisher",
+                    operation: event.event,
+                    phase: "publish",
+                });
+                return err(error);
             }
         })());
     }
@@ -1229,10 +1275,22 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         __classPrivateFieldGet(this, _Trellis_log, "f").trace({ method: String(method) }, `Calling ${method.toString()}.`);
         const msg = encodeRuntimeSchema(ctx.input, input).take();
         if (isErr(msg)) {
+            recordRuntimeError(msg.error, {
+                surface: "rpc",
+                direction: "client",
+                operation: method,
+                phase: "request_encoding",
+            });
             return msg;
         }
         const subject = this.template(ctx.subject, input).take();
         if (isErr(subject)) {
+            recordRuntimeError(subject.error, {
+                surface: "rpc",
+                direction: "client",
+                operation: method,
+                phase: "request_encoding",
+            });
             return subject;
         }
         const span = startClientSpan(method, subject);
@@ -1254,30 +1312,50 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             });
             const response = msgResult.take();
             if (isErr(response)) {
+                recordRuntimeError(response.error, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "request_send",
+                });
                 return response;
             }
             if (response.headers?.get("status") === "error") {
                 const json = safeJson(response).take();
                 if (isErr(json)) {
-                    return err(requestFailedTransportError({
+                    const error = requestFailedTransportError({
                         code: "trellis.request.invalid_response",
                         message: "Trellis returned an invalid response.",
                         hint: "Retry the request. If it keeps happening, check the Trellis capability handling this request.",
                         method,
                         subject,
                         cause: json.error.cause,
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "client",
+                        operation: method,
+                        phase: "response_decoding",
+                    });
+                    return err(error);
                 }
                 const errorData = parse(TrellisErrorDataSchema, json).take();
                 if (isErr(errorData)) {
-                    return err(requestFailedTransportError({
+                    const error = requestFailedTransportError({
                         code: "trellis.request.invalid_response",
                         message: "Trellis returned an invalid response.",
                         hint: "Retry the request. If it keeps happening, check the Trellis capability handling this request.",
                         method,
                         subject,
                         cause: errorData.error,
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "client",
+                        operation: method,
+                        phase: "response_decoding",
+                    });
+                    return err(error);
                 }
                 const declaredErrorTypes = Array.isArray(ctx.declaredErrorTypes)
                     ? ctx.declaredErrorTypes.filter((value) => typeof value === "string")
@@ -1288,25 +1366,50 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 const reconstructed = reconstructDeclaredRpcError(declaredErrorTypes, runtimeErrors, errorData, json);
                 if (reconstructed) {
                     await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_handleBrowserAuthRequired).call(this, reconstructed);
+                    recordRuntimeError(new RemoteError({ error: errorData }), {
+                        surface: "rpc",
+                        direction: "client",
+                        operation: method,
+                        phase: "remote_error",
+                    });
                     return err(reconstructed);
                 }
                 const remoteError = new RemoteError({ error: errorData });
                 await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_handleBrowserAuthRequired).call(this, remoteError);
+                recordRuntimeError(remoteError, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "remote_error",
+                });
                 return err(remoteError);
             }
             const json = safeJson(response).take();
             if (isErr(json)) {
-                return err(requestFailedTransportError({
+                const error = requestFailedTransportError({
                     code: "trellis.request.invalid_response",
                     message: "Trellis returned an invalid response.",
                     hint: "Retry the request. If it keeps happening, check the Trellis capability handling this request.",
                     method,
                     subject,
                     cause: json.error.cause,
-                }));
+                });
+                recordRuntimeError(error, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "response_decoding",
+                });
+                return err(error);
             }
             const outputResult = parseRuntimeSchema(ctx.output, json).take();
             if (isErr(outputResult)) {
+                recordRuntimeError(outputResult.error, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "response_decoding",
+                });
                 return err(outputResult.error);
             }
             return ok(outputResult);
@@ -1335,6 +1438,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     message: unexpected.message,
                 });
                 span.recordException(unexpected);
+                recordRuntimeError(unexpected, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "unexpected",
+                });
                 return err(unexpected);
             }
             finally {
@@ -1403,11 +1512,25 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
 }, _Trellis_subscribeFeed = function _Trellis_subscribeFeed(feed, descriptor, input, opts) {
     return AsyncResult.from((async () => {
         const payload = encodeRuntimeSchema(descriptor.input, input).take();
-        if (isErr(payload))
+        if (isErr(payload)) {
+            recordRuntimeError(payload.error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "request_encoding",
+            });
             return payload;
+        }
         const subject = this.template(descriptor.subject, input).take();
-        if (isErr(subject))
+        if (isErr(subject)) {
+            recordRuntimeError(subject.error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "request_template",
+            });
             return subject;
+        }
         const authHeaders = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_createProof).call(this, subject, payload);
         const headers = natsHeaders();
         headers.set("session-key", this.auth.sessionKey);
@@ -1427,13 +1550,20 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         catch (cause) {
             opts?.signal?.removeEventListener("abort", abort);
             sub.unsubscribe();
-            return err(createTransportError({
+            const error = createTransportError({
                 code: "trellis.feed.subscribe_failed",
                 message: "Trellis could not subscribe to the feed.",
                 hint: "Retry the subscription. If it keeps failing, check Trellis runtime health.",
                 cause,
                 context: { feed, subject },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "request_send",
+            });
+            return err(error);
         }
         let timeoutId;
         let abortHandler;
@@ -1459,7 +1589,7 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         if (firstFrame === "timeout" || firstFrame === "aborted") {
             opts?.signal?.removeEventListener("abort", abort);
             sub.unsubscribe();
-            return err(createTransportError({
+            const error = createTransportError({
                 code: firstFrame === "timeout"
                     ? "trellis.feed.subscribe_timeout"
                     : "trellis.feed.subscribe_aborted",
@@ -1470,28 +1600,49 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     ? "Check that the target service is running and has the current deployment digest, then retry."
                     : "Retry the subscription if the feed is still needed.",
                 context: { feed, subject },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "handshake",
+            });
+            return err(error);
         }
         if (firstFrame.done) {
             opts?.signal?.removeEventListener("abort", abort);
             sub.unsubscribe();
-            return err(createTransportError({
+            const error = createTransportError({
                 code: "trellis.feed.subscribe_closed",
                 message: "Trellis closed the feed before acknowledging it.",
                 hint: "Retry the subscription. If it keeps failing, check Trellis runtime health.",
                 context: { feed, subject },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "handshake",
+            });
+            return err(error);
         }
         const firstMessage = firstFrame.value;
         if (firstMessage.headers?.get("status") === "error") {
             opts?.signal?.removeEventListener("abort", abort);
             sub.unsubscribe();
-            return err(createTransportError({
+            const error = createTransportError({
                 code: "trellis.feed.failed",
                 message: "Trellis rejected the feed subscription.",
                 hint: "Retry the subscription. If it keeps failing, check Trellis runtime health and permissions.",
                 context: { feed, subject, frame: firstMessage.string() },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "remote_error",
+            });
+            return err(error);
         }
         const firstEvent = firstMessage.headers?.get("feed-status") === "ready"
             ? undefined
@@ -1501,19 +1652,40 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             try {
                 const parseFeedFrame = (msg) => {
                     if (msg.headers?.get("status") === "error") {
-                        throw createTransportError({
+                        const error = createTransportError({
                             code: "trellis.feed.failed",
                             message: "Trellis stopped the feed.",
                             hint: "Retry the subscription. If it keeps failing, check Trellis runtime health.",
                             context: { feed, subject, frame: msg.string() },
                         });
+                        recordRuntimeError(error, {
+                            surface: "feed",
+                            direction: "client",
+                            operation: feed,
+                            phase: "remote_error",
+                        });
+                        throw error;
                     }
                     const json = safeJson(msg).take();
-                    if (isErr(json))
+                    if (isErr(json)) {
+                        recordRuntimeError(json.error, {
+                            surface: "feed",
+                            direction: "client",
+                            operation: feed,
+                            phase: "event_decoding",
+                        });
                         throw json.error;
+                    }
                     const parsed = parseRuntimeSchema(eventSchema, json).take();
-                    if (isErr(parsed))
+                    if (isErr(parsed)) {
+                        recordRuntimeError(parsed.error, {
+                            surface: "feed",
+                            direction: "client",
+                            operation: feed,
+                            phase: "event_validation",
+                        });
                         throw parsed.error;
+                    }
                     return parsed;
                 };
                 if (firstEvent)
@@ -1541,13 +1713,20 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         await this.nats.flush();
     }
     catch (cause) {
-        throw createTransportError({
+        const error = createTransportError({
             code: "trellis.feed.listen_failed",
             message: "Trellis could not listen for feed requests.",
             hint: "Check the service deployment digest and runtime permissions, then restart the service.",
             cause,
             context: { feed, subject },
         });
+        recordRuntimeError(error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "listen",
+        });
+        throw error;
     }
     const task = AsyncResult.try(async () => {
         for await (const msg of sub) {
@@ -1568,6 +1747,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         contractDigest: this.contractDigest,
                         traceId: traceIdFromTraceparent(msg.headers?.get("traceparent")),
                     });
+                    recordRuntimeError(error, {
+                        surface: "feed",
+                        direction: "server",
+                        operation: feed,
+                        phase: "handler_throw",
+                    });
                     __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_respondWithError).call(this, msg, error);
                 }
             })();
@@ -1576,11 +1761,25 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
     __classPrivateFieldGet(this, _Trellis_tasks, "f").add(`feed:${feed}`, task);
 }, _Trellis_processFeedMessage = async function _Trellis_processFeedMessage(feed, descriptor, msg, handler) {
     const json = safeJson(msg).take();
-    if (isErr(json))
+    if (isErr(json)) {
+        recordRuntimeError(json.error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "request_decoding",
+        });
         return json;
+    }
     const parsed = parseRuntimeSchema(descriptor.input, json).take();
-    if (isErr(parsed))
+    if (isErr(parsed)) {
+        recordRuntimeError(parsed.error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "input_validation",
+        });
         return parsed;
+    }
     const caller = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_authenticateFeedRequest).call(this, {
         feed,
         subject: msg.subject,
@@ -1589,12 +1788,26 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         requiredCapabilities: descriptor.subscribeCapabilities,
     });
     const callerValue = caller.take();
-    if (isErr(callerValue))
+    if (isErr(callerValue)) {
+        recordRuntimeError(callerValue.error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "auth",
+        });
         return callerValue;
+    }
     if (!msg.reply) {
-        return err(new UnexpectedError({
+        const error = new UnexpectedError({
             context: { feed, reason: "missing_reply" },
-        }));
+        });
+        recordRuntimeError(error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "handshake",
+        });
+        return err(error);
     }
     const readyHeaders = natsHeaders();
     readyHeaders.set("feed-status", "ready");
@@ -1608,15 +1821,44 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             signal: controller.signal,
             emit: (event) => AsyncResult.from((async () => {
                 const payload = encodeRuntimeSchema(descriptor.event, event).take();
-                if (isErr(payload))
+                if (isErr(payload)) {
+                    recordRuntimeError(payload.error, {
+                        surface: "feed",
+                        direction: "server",
+                        operation: feed,
+                        phase: "event_encoding",
+                    });
                     return payload;
+                }
                 if (!msg.reply) {
-                    return err(new UnexpectedError({
+                    const error = new UnexpectedError({
                         context: { feed, reason: "missing_reply" },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "feed",
+                        direction: "server",
+                        operation: feed,
+                        phase: "event_publish",
+                    });
+                    return err(error);
+                }
+                try {
+                    this.nats.publish(msg.reply, payload);
+                    await this.nats.flush();
+                }
+                catch (cause) {
+                    const error = new UnexpectedError({
+                        cause,
+                        context: { feed },
+                    });
+                    recordRuntimeError(error, {
+                        surface: "feed",
+                        direction: "server",
+                        operation: feed,
+                        phase: "event_publish",
+                    });
+                    return err(error);
                 }
-                this.nats.publish(msg.reply, payload);
-                await this.nats.flush();
                 return ok(undefined);
             })()),
         });
@@ -1624,14 +1866,21 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             ? handlerResult.take()
             : handlerResult;
         if (isErr(handlerOutcome)) {
-            return err(annotateHandlerBoundaryError(handlerOutcome.error, {
+            const error = annotateHandlerBoundaryError(handlerOutcome.error, {
                 feed,
                 requestId: msg.headers?.get("request-id"),
                 service: this.name,
                 contractId: this.contractId,
                 contractDigest: this.contractDigest,
                 traceId: traceIdFromTraceparent(msg.headers?.get("traceparent")),
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "server",
+                operation: feed,
+                phase: "handler_result",
+            });
+            return err(error);
         }
         return ok(undefined);
     }
@@ -1700,6 +1949,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     code: SpanStatusCode.ERROR,
                     message: "Failed to parse JSON",
                 });
+                recordRuntimeError(jsonData.error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "parse",
+                });
                 return jsonData;
             }
             const parsedInput = parseRuntimeSchema(ctx.input, jsonData).take();
@@ -1708,6 +1963,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     code: SpanStatusCode.ERROR,
                     message: "Input validation failed",
                 });
+                recordRuntimeError(parsedInput.error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "input_validation",
+                });
                 return parsedInput;
             }
             let caller;
@@ -1733,7 +1994,14 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         code: SpanStatusCode.ERROR,
                         message: "Missing session-key",
                     });
-                    return err(new AuthError({ reason: "missing_session_key" }));
+                    const error = new AuthError({ reason: "missing_session_key" });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 if (!proof) {
                     __classPrivateFieldGet(this, _Trellis_log, "f").warn({ method }, "Missing proof in request");
@@ -1741,11 +2009,25 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         code: SpanStatusCode.ERROR,
                         message: "Missing proof",
                     });
-                    return err(new AuthError({ reason: "missing_proof" }));
+                    const error = new AuthError({ reason: "missing_proof" });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 const iat = Number(iatHeader);
                 if (!Number.isSafeInteger(iat) || !requestId) {
-                    return err(new AuthError({ reason: "invalid_signature" }));
+                    const error = new AuthError({ reason: "invalid_signature" });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 // Verify proof signature locally using the raw request bytes we received.
                 const payloadBytes = msg.data ?? new Uint8Array();
@@ -1764,10 +2046,17 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         code: SpanStatusCode.ERROR,
                         message: "Invalid signature",
                     });
-                    return err(new AuthError({
+                    const error = new AuthError({
                         reason: "invalid_signature",
                         context: { sessionKey },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 let auth;
                 for (let attempt = 0; attempt < DEFAULT_AUTH_VALIDATE_SESSION_RETRY_ATTEMPTS; attempt++) {
@@ -1793,9 +2082,16 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     await sleep(DEFAULT_AUTH_VALIDATE_SESSION_RETRY_MS * (attempt + 1));
                 }
                 if (!auth) {
-                    return err(new UnexpectedError({
+                    const error = new UnexpectedError({
                         context: { reason: "missing_auth_validate_result" },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 if (auth instanceof Error) {
                     __classPrivateFieldGet(this, _Trellis_log, "f").warn({
@@ -1811,22 +2107,42 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         message: "Auth.Requests.Validate failed",
                     });
                     if (auth instanceof BaseError) {
+                        recordRuntimeError(auth, {
+                            surface: "rpc",
+                            direction: "server",
+                            operation: String(method),
+                            phase: "auth",
+                        });
                         return err(auth);
                     }
-                    return err(new UnexpectedError({ cause: auth }));
+                    const error = new UnexpectedError({ cause: auth });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 if (!auth.allowed) {
                     span.setStatus({
                         code: SpanStatusCode.ERROR,
                         message: "Insufficient permissions",
                     });
-                    return err(new AuthError({
+                    const error = new AuthError({
                         reason: "insufficient_permissions",
                         context: {
                             requiredCapabilities: ctx.callerCapabilities,
                             userCapabilities: auth.caller.capabilities,
                         },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 if (typeof msg.reply !== "string" ||
                     !msg.reply.startsWith(`${auth.inboxPrefix}.`)) {
@@ -1834,10 +2150,17 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         code: SpanStatusCode.ERROR,
                         message: "Reply subject mismatch",
                     });
-                    return err(new AuthError({
+                    const error = new AuthError({
                         reason: "reply_subject_mismatch",
                         context: { expected: auth.inboxPrefix, actual: msg.reply },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return err(error);
                 }
                 caller = auth.caller;
             }
@@ -1885,6 +2208,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     message: error.message,
                 });
                 span.recordException(error);
+                recordRuntimeError(error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "handler_throw",
+                });
                 return err(error);
             }
             const handlerResult = handlerResultWrapped.take();
@@ -1910,6 +2239,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     code: SpanStatusCode.ERROR,
                     message: error.message,
                 });
+                recordRuntimeError(error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "handler_result",
+                });
                 return err(error);
             }
             const encoded = encodeSchema(ctx.output, handlerOutcome).take();
@@ -1918,6 +2253,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     code: SpanStatusCode.ERROR,
                     message: "Output encoding failed",
                 });
+                recordRuntimeError(encoded.error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "output_encoding",
+                });
                 return encoded;
             }
             span.setStatus({ code: SpanStatusCode.OK });
@@ -1994,6 +2335,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             const m = parsedEvent.take();
             if (isErr(m)) {
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({ error: m.error }, "Event validation failed");
+                recordRuntimeError(m.error, {
+                    surface: "event",
+                    direction: "consumer",
+                    operation: String(event),
+                    phase: "input_validation",
+                });
                 continue;
             }
             const handlerResult = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_invokeEventHandler).call(this, {
@@ -2005,6 +2352,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             });
             const handlerValue = handlerResult.take();
             if (isErr(handlerValue)) {
+                recordRuntimeError(handlerValue.error, {
+                    surface: "event",
+                    direction: "consumer",
+                    operation: String(event),
+                    phase: "handler_result",
+                });
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({
                     error: handlerValue.error.toSerializable(),
                     event,
@@ -2204,6 +2557,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 const eventPayload = parsedEvent.take();
                 if (isErr(eventPayload)) {
                     __classPrivateFieldGet(this, _Trellis_log, "f").error({ error: eventPayload.error }, "Event validation failed");
+                    recordRuntimeError(eventPayload.error, {
+                        surface: "event",
+                        direction: "consumer",
+                        operation: String(registration.event),
+                        phase: "input_validation",
+                    });
                     msg.term();
                     failed = true;
                     break;
@@ -2218,6 +2577,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 });
                 const handlerValue = handlerResult.take();
                 if (isErr(handlerValue)) {
+                    recordRuntimeError(handlerValue.error, {
+                        surface: "event",
+                        direction: "consumer",
+                        operation: String(registration.event),
+                        phase: "handler_result",
+                    });
                     __classPrivateFieldGet(this, _Trellis_log, "f").error({
                         error: handlerValue.error.toSerializable(),
                         event: registration.event,
@@ -2315,6 +2680,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         code: SpanStatusCode.ERROR,
                         message: response.error.message,
                     });
+                    recordRuntimeError(response.error, {
+                        surface: "operation",
+                        direction: "client",
+                        operation: "requestJson",
+                        phase: "request_send",
+                    });
                     return response;
                 }
                 const json = safeJson(response).take();
@@ -2330,6 +2701,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         code: SpanStatusCode.ERROR,
                         message: error.message,
                     });
+                    recordRuntimeError(error, {
+                        surface: "operation",
+                        direction: "client",
+                        operation: "requestJson",
+                        phase: "response_decoding",
+                    });
                     return err(error);
                 }
                 span.setStatus({ code: SpanStatusCode.OK });
@@ -2342,6 +2719,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     message: error.message,
                 });
                 span.recordException(error);
+                recordRuntimeError(error, {
+                    surface: "operation",
+                    direction: "client",
+                    operation: "requestJson",
+                    phase: "unexpected",
+                });
                 return err(error);
             }
             finally {
@@ -2369,35 +2752,56 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         }
         catch (cause) {
             sub.unsubscribe();
-            return err(createTransportError({
+            const error = createTransportError({
                 code: "trellis.watch.failed",
                 message: "Trellis could not start the operation watch.",
                 hint: "Retry watching the operation. If it keeps failing, reconnect to Trellis and try again.",
                 cause,
                 context: { subject },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "operation",
+                direction: "client",
+                operation: "watchJson",
+                phase: "request_send",
+            });
+            return err(error);
         }
         return ok((async function* () {
             try {
                 for await (const msg of sub) {
                     if (msg.headers?.get("status") === "error") {
-                        yield err(createTransportError({
+                        const error = createTransportError({
                             code: "trellis.watch.failed",
                             message: "Trellis stopped the operation watch.",
                             hint: "Retry watching the operation. If it keeps happening, reconnect to Trellis and try again.",
                             context: { subject, frame: msg.string() },
-                        }));
+                        });
+                        recordRuntimeError(error, {
+                            surface: "operation",
+                            direction: "client",
+                            operation: "watchJson",
+                            phase: "remote_error",
+                        });
+                        yield err(error);
                         continue;
                     }
                     const json = safeJson(msg).take();
                     if (isErr(json)) {
-                        yield err(createTransportError({
+                        const error = createTransportError({
                             code: "trellis.watch.invalid_response",
                             message: "Trellis returned an invalid watch update.",
                             hint: "Retry watching the operation. If it keeps happening, reconnect to Trellis and try again.",
                             cause: json.error.cause,
                             context: { subject },
-                        }));
+                        });
+                        recordRuntimeError(error, {
+                            surface: "operation",
+                            direction: "client",
+                            operation: "watchJson",
+                            phase: "response_decoding",
+                        });
+                        yield err(error);
                         continue;
                     }
                     yield ok(json);