@qlever-llc/trellis 0.10.11 → 0.10.12

package/script/trellis.js

@@ -29,7 +29,7 @@ const jetstream_1 = require("@nats-io/jetstream");
 const nats_core_1 = require("@nats-io/nats-core");
 const mod_js_1 = require("./contract_support/mod.js");
 const result_1 = require("@qlever-llc/result");
-const tracing_js_1 = require("./tracing.js");
+const mod_js_2 = require("./telemetry/mod.js");
 const typebox_1 = require("typebox");
 const value_1 = require("typebox/value");
 const ulid_1 = require("ulid");
@@ -220,6 +220,12 @@ function annotateHandlerBoundaryError(cause, context) {
     error.withTraceId(context.traceId);
     return error;
 }
+function recordRuntimeError(error, attributes) {
+    (0, mod_js_2.recordTrellisError)(error, {
+        messagingSystem: "nats",
+        ...attributes,
+    });
+}
 const DurableOperationSignalSchema = typebox_1.Type.Object({
     operationId: typebox_1.Type.String(),
     sequence: typebox_1.Type.Number(),
@@ -950,14 +956,27 @@ class Trellis {
             const eventName = event;
             const ctx = this.api["events"][eventName];
             if (!ctx) {
-                return (0, result_1.err)(new index_js_1.UnexpectedError({
+                const error = new index_js_1.UnexpectedError({
                     cause: __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_unknownApiError).call(this, "event", event.toString()),
                     context: { event: event.toString() },
-                }));
+                });
+                recordRuntimeError(error, {
+                    surface: "event",
+                    direction: "publisher",
+                    operation: event,
+                    phase: "prepare",
+                });
+                return (0, result_1.err)(error);
             }
             const subject = this.template(ctx.subject, data).take();
             if ((0, result_1.isErr)(subject)) {
                 globals_js_1.logger.error({ err: subject.error }, "Failed to template event.");
+                recordRuntimeError(subject.error, {
+                    surface: "event",
+                    direction: "publisher",
+                    operation: event,
+                    phase: "request_encoding",
+                });
                 return subject;
             }
             const header = {
@@ -971,11 +990,18 @@ class Trellis {
             const msg = (0, codec_js_1.encodeSchema)(ctx.event, payload).take();
             if ((0, result_1.isErr)(msg)) {
                 globals_js_1.logger.error({ err: msg.error }, "Failed to encode event.");
-                return (0, result_1.err)(new index_js_1.UnexpectedError({ cause: msg.error }));
+                const error = new index_js_1.UnexpectedError({ cause: msg.error });
+                recordRuntimeError(error, {
+                    surface: "event",
+                    direction: "publisher",
+                    operation: event,
+                    phase: "request_encoding",
+                });
+                return (0, result_1.err)(error);
             }
             const headers = (0, nats_core_1.headers)();
             headers.set("Nats-Msg-Id", header.id);
-            (0, tracing_js_1.injectTraceContext)((0, tracing_js_1.createNatsHeaderCarrier)(headers));
+            (0, mod_js_2.injectTraceContext)((0, mod_js_2.createNatsHeaderCarrier)(headers));
             const headerRecord = {};
             for (const [key, value] of headers) {
                 headerRecord[key] = value.join(",");
@@ -989,7 +1015,17 @@ class Trellis {
             }));
         }
         catch (cause) {
-            return (0, result_1.err)(new index_js_1.UnexpectedError({ cause, context: { event: event.toString() } }));
+            const error = new index_js_1.UnexpectedError({
+                cause,
+                context: { event: event.toString() },
+            });
+            recordRuntimeError(error, {
+                surface: "event",
+                direction: "publisher",
+                operation: event,
+                phase: "prepare",
+            });
+            return (0, result_1.err)(error);
         }
     }
     /**
@@ -1008,7 +1044,17 @@ class Trellis {
                 return (0, result_1.ok)(undefined);
             }
             catch (cause) {
-                return (0, result_1.err)(new index_js_1.UnexpectedError({ cause, context: { event: event.event } }));
+                const error = new index_js_1.UnexpectedError({
+                    cause,
+                    context: { event: event.event },
+                });
+                recordRuntimeError(error, {
+                    surface: "event",
+                    direction: "publisher",
+                    operation: event.event,
+                    phase: "publish",
+                });
+                return (0, result_1.err)(error);
             }
         })());
     }
@@ -1245,13 +1291,25 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         __classPrivateFieldGet(this, _Trellis_log, "f").trace({ method: String(method) }, `Calling ${method.toString()}.`);
         const msg = encodeRuntimeSchema(ctx.input, input).take();
         if ((0, result_1.isErr)(msg)) {
+            recordRuntimeError(msg.error, {
+                surface: "rpc",
+                direction: "client",
+                operation: method,
+                phase: "request_encoding",
+            });
             return msg;
         }
         const subject = this.template(ctx.subject, input).take();
         if ((0, result_1.isErr)(subject)) {
+            recordRuntimeError(subject.error, {
+                surface: "rpc",
+                direction: "client",
+                operation: method,
+                phase: "request_encoding",
+            });
             return subject;
         }
-        const span = (0, tracing_js_1.startClientSpan)(method, subject);
+        const span = (0, mod_js_2.startClientSpan)(method, subject);
         const attempt = async () => {
             const authHeaders = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_createProof).call(this, subject, msg);
             const headers = (0, nats_core_1.headers)();
@@ -1259,7 +1317,7 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             headers.set("proof", authHeaders.proof);
             headers.set("iat", String(authHeaders.iat));
             headers.set("request-id", authHeaders.requestId);
-            (0, tracing_js_1.injectTraceContext)((0, tracing_js_1.createNatsHeaderCarrier)(headers), span);
+            (0, mod_js_2.injectTraceContext)((0, mod_js_2.createNatsHeaderCarrier)(headers), span);
             const msgResult = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestMessageWithRetry).call(this, {
                 method,
                 subject,
@@ -1270,30 +1328,50 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             });
             const response = msgResult.take();
             if ((0, result_1.isErr)(response)) {
+                recordRuntimeError(response.error, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "request_send",
+                });
                 return response;
             }
             if (response.headers?.get("status") === "error") {
                 const json = safeJson(response).take();
                 if ((0, result_1.isErr)(json)) {
-                    return (0, result_1.err)(requestFailedTransportError({
+                    const error = requestFailedTransportError({
                         code: "trellis.request.invalid_response",
                         message: "Trellis returned an invalid response.",
                         hint: "Retry the request. If it keeps happening, check the Trellis capability handling this request.",
                         method,
                         subject,
                         cause: json.error.cause,
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "client",
+                        operation: method,
+                        phase: "response_decoding",
+                    });
+                    return (0, result_1.err)(error);
                 }
                 const errorData = (0, codec_js_1.parse)(TrellisError_js_1.TrellisErrorDataSchema, json).take();
                 if ((0, result_1.isErr)(errorData)) {
-                    return (0, result_1.err)(requestFailedTransportError({
+                    const error = requestFailedTransportError({
                         code: "trellis.request.invalid_response",
                         message: "Trellis returned an invalid response.",
                         hint: "Retry the request. If it keeps happening, check the Trellis capability handling this request.",
                         method,
                         subject,
                         cause: errorData.error,
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "client",
+                        operation: method,
+                        phase: "response_decoding",
+                    });
+                    return (0, result_1.err)(error);
                 }
                 const declaredErrorTypes = Array.isArray(ctx.declaredErrorTypes)
                     ? ctx.declaredErrorTypes.filter((value) => typeof value === "string")
@@ -1304,41 +1382,66 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 const reconstructed = reconstructDeclaredRpcError(declaredErrorTypes, runtimeErrors, errorData, json);
                 if (reconstructed) {
                     await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_handleBrowserAuthRequired).call(this, reconstructed);
+                    recordRuntimeError(new RemoteError_js_1.RemoteError({ error: errorData }), {
+                        surface: "rpc",
+                        direction: "client",
+                        operation: method,
+                        phase: "remote_error",
+                    });
                     return (0, result_1.err)(reconstructed);
                 }
                 const remoteError = new RemoteError_js_1.RemoteError({ error: errorData });
                 await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_handleBrowserAuthRequired).call(this, remoteError);
+                recordRuntimeError(remoteError, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "remote_error",
+                });
                 return (0, result_1.err)(remoteError);
             }
             const json = safeJson(response).take();
             if ((0, result_1.isErr)(json)) {
-                return (0, result_1.err)(requestFailedTransportError({
+                const error = requestFailedTransportError({
                     code: "trellis.request.invalid_response",
                     message: "Trellis returned an invalid response.",
                     hint: "Retry the request. If it keeps happening, check the Trellis capability handling this request.",
                     method,
                     subject,
                     cause: json.error.cause,
-                }));
+                });
+                recordRuntimeError(error, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "response_decoding",
+                });
+                return (0, result_1.err)(error);
             }
             const outputResult = parseRuntimeSchema(ctx.output, json).take();
             if ((0, result_1.isErr)(outputResult)) {
+                recordRuntimeError(outputResult.error, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "response_decoding",
+                });
                 return (0, result_1.err)(outputResult.error);
             }
             return (0, result_1.ok)(outputResult);
         };
-        return await (0, tracing_js_1.withSpanAsync)(span, async () => {
+        return await (0, mod_js_2.withSpanAsync)(span, async () => {
             try {
                 const result = await attempt();
                 const value = result.take();
                 if ((0, result_1.isErr)(value)) {
                     span.setStatus({
-                        code: tracing_js_1.SpanStatusCode.ERROR,
+                        code: mod_js_2.SpanStatusCode.ERROR,
                         message: value.error.message,
                     });
                 }
                 else {
-                    span.setStatus({ code: tracing_js_1.SpanStatusCode.OK });
+                    span.setStatus({ code: mod_js_2.SpanStatusCode.OK });
                 }
                 return result;
             }
@@ -1347,10 +1450,16 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     ? cause
                     : new index_js_1.UnexpectedError({ cause });
                 span.setStatus({
-                    code: tracing_js_1.SpanStatusCode.ERROR,
+                    code: mod_js_2.SpanStatusCode.ERROR,
                     message: unexpected.message,
                 });
                 span.recordException(unexpected);
+                recordRuntimeError(unexpected, {
+                    surface: "rpc",
+                    direction: "client",
+                    operation: method,
+                    phase: "unexpected",
+                });
                 return (0, result_1.err)(unexpected);
             }
             finally {
@@ -1419,18 +1528,32 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
 }, _Trellis_subscribeFeed = function _Trellis_subscribeFeed(feed, descriptor, input, opts) {
     return result_1.AsyncResult.from((async () => {
         const payload = encodeRuntimeSchema(descriptor.input, input).take();
-        if ((0, result_1.isErr)(payload))
+        if ((0, result_1.isErr)(payload)) {
+            recordRuntimeError(payload.error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "request_encoding",
+            });
             return payload;
+        }
         const subject = this.template(descriptor.subject, input).take();
-        if ((0, result_1.isErr)(subject))
+        if ((0, result_1.isErr)(subject)) {
+            recordRuntimeError(subject.error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "request_template",
+            });
             return subject;
+        }
         const authHeaders = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_createProof).call(this, subject, payload);
         const headers = (0, nats_core_1.headers)();
         headers.set("session-key", this.auth.sessionKey);
         headers.set("proof", authHeaders.proof);
         headers.set("iat", String(authHeaders.iat));
         headers.set("request-id", authHeaders.requestId);
-        (0, tracing_js_1.injectTraceContext)((0, tracing_js_1.createNatsHeaderCarrier)(headers));
+        (0, mod_js_2.injectTraceContext)((0, mod_js_2.createNatsHeaderCarrier)(headers));
         const inbox = (0, nats_core_1.createInbox)(`_INBOX.${this.auth.sessionKey.slice(0, 16)}`);
         const sub = this.nats.subscribe(inbox);
         const iterator = sub[Symbol.asyncIterator]();
@@ -1443,13 +1566,20 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         catch (cause) {
             opts?.signal?.removeEventListener("abort", abort);
             sub.unsubscribe();
-            return (0, result_1.err)(createTransportError({
+            const error = createTransportError({
                 code: "trellis.feed.subscribe_failed",
                 message: "Trellis could not subscribe to the feed.",
                 hint: "Retry the subscription. If it keeps failing, check Trellis runtime health.",
                 cause,
                 context: { feed, subject },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "request_send",
+            });
+            return (0, result_1.err)(error);
         }
         let timeoutId;
         let abortHandler;
@@ -1475,7 +1605,7 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         if (firstFrame === "timeout" || firstFrame === "aborted") {
             opts?.signal?.removeEventListener("abort", abort);
             sub.unsubscribe();
-            return (0, result_1.err)(createTransportError({
+            const error = createTransportError({
                 code: firstFrame === "timeout"
                     ? "trellis.feed.subscribe_timeout"
                     : "trellis.feed.subscribe_aborted",
@@ -1486,28 +1616,49 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     ? "Check that the target service is running and has the current deployment digest, then retry."
                     : "Retry the subscription if the feed is still needed.",
                 context: { feed, subject },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "handshake",
+            });
+            return (0, result_1.err)(error);
         }
         if (firstFrame.done) {
             opts?.signal?.removeEventListener("abort", abort);
             sub.unsubscribe();
-            return (0, result_1.err)(createTransportError({
+            const error = createTransportError({
                 code: "trellis.feed.subscribe_closed",
                 message: "Trellis closed the feed before acknowledging it.",
                 hint: "Retry the subscription. If it keeps failing, check Trellis runtime health.",
                 context: { feed, subject },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "handshake",
+            });
+            return (0, result_1.err)(error);
         }
         const firstMessage = firstFrame.value;
         if (firstMessage.headers?.get("status") === "error") {
             opts?.signal?.removeEventListener("abort", abort);
             sub.unsubscribe();
-            return (0, result_1.err)(createTransportError({
+            const error = createTransportError({
                 code: "trellis.feed.failed",
                 message: "Trellis rejected the feed subscription.",
                 hint: "Retry the subscription. If it keeps failing, check Trellis runtime health and permissions.",
                 context: { feed, subject, frame: firstMessage.string() },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "client",
+                operation: feed,
+                phase: "remote_error",
+            });
+            return (0, result_1.err)(error);
         }
         const firstEvent = firstMessage.headers?.get("feed-status") === "ready"
             ? undefined
@@ -1517,19 +1668,40 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             try {
                 const parseFeedFrame = (msg) => {
                     if (msg.headers?.get("status") === "error") {
-                        throw createTransportError({
+                        const error = createTransportError({
                             code: "trellis.feed.failed",
                             message: "Trellis stopped the feed.",
                             hint: "Retry the subscription. If it keeps failing, check Trellis runtime health.",
                             context: { feed, subject, frame: msg.string() },
                         });
+                        recordRuntimeError(error, {
+                            surface: "feed",
+                            direction: "client",
+                            operation: feed,
+                            phase: "remote_error",
+                        });
+                        throw error;
                     }
                     const json = safeJson(msg).take();
-                    if ((0, result_1.isErr)(json))
+                    if ((0, result_1.isErr)(json)) {
+                        recordRuntimeError(json.error, {
+                            surface: "feed",
+                            direction: "client",
+                            operation: feed,
+                            phase: "event_decoding",
+                        });
                         throw json.error;
+                    }
                     const parsed = parseRuntimeSchema(eventSchema, json).take();
-                    if ((0, result_1.isErr)(parsed))
+                    if ((0, result_1.isErr)(parsed)) {
+                        recordRuntimeError(parsed.error, {
+                            surface: "feed",
+                            direction: "client",
+                            operation: feed,
+                            phase: "event_validation",
+                        });
                         throw parsed.error;
+                    }
                     return parsed;
                 };
                 if (firstEvent)
@@ -1557,13 +1729,20 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         await this.nats.flush();
     }
     catch (cause) {
-        throw createTransportError({
+        const error = createTransportError({
             code: "trellis.feed.listen_failed",
             message: "Trellis could not listen for feed requests.",
             hint: "Check the service deployment digest and runtime permissions, then restart the service.",
             cause,
             context: { feed, subject },
         });
+        recordRuntimeError(error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "listen",
+        });
+        throw error;
     }
     const task = result_1.AsyncResult.try(async () => {
         for await (const msg of sub) {
@@ -1584,6 +1763,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         contractDigest: this.contractDigest,
                         traceId: traceIdFromTraceparent(msg.headers?.get("traceparent")),
                     });
+                    recordRuntimeError(error, {
+                        surface: "feed",
+                        direction: "server",
+                        operation: feed,
+                        phase: "handler_throw",
+                    });
                     __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_respondWithError).call(this, msg, error);
                 }
             })();
@@ -1592,11 +1777,25 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
     __classPrivateFieldGet(this, _Trellis_tasks, "f").add(`feed:${feed}`, task);
 }, _Trellis_processFeedMessage = async function _Trellis_processFeedMessage(feed, descriptor, msg, handler) {
     const json = safeJson(msg).take();
-    if ((0, result_1.isErr)(json))
+    if ((0, result_1.isErr)(json)) {
+        recordRuntimeError(json.error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "request_decoding",
+        });
         return json;
+    }
     const parsed = parseRuntimeSchema(descriptor.input, json).take();
-    if ((0, result_1.isErr)(parsed))
+    if ((0, result_1.isErr)(parsed)) {
+        recordRuntimeError(parsed.error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "input_validation",
+        });
         return parsed;
+    }
     const caller = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_authenticateFeedRequest).call(this, {
         feed,
         subject: msg.subject,
@@ -1605,12 +1804,26 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         requiredCapabilities: descriptor.subscribeCapabilities,
     });
     const callerValue = caller.take();
-    if ((0, result_1.isErr)(callerValue))
+    if ((0, result_1.isErr)(callerValue)) {
+        recordRuntimeError(callerValue.error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "auth",
+        });
         return callerValue;
+    }
     if (!msg.reply) {
-        return (0, result_1.err)(new index_js_1.UnexpectedError({
+        const error = new index_js_1.UnexpectedError({
             context: { feed, reason: "missing_reply" },
-        }));
+        });
+        recordRuntimeError(error, {
+            surface: "feed",
+            direction: "server",
+            operation: feed,
+            phase: "handshake",
+        });
+        return (0, result_1.err)(error);
     }
     const readyHeaders = (0, nats_core_1.headers)();
     readyHeaders.set("feed-status", "ready");
@@ -1624,15 +1837,44 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             signal: controller.signal,
             emit: (event) => result_1.AsyncResult.from((async () => {
                 const payload = encodeRuntimeSchema(descriptor.event, event).take();
-                if ((0, result_1.isErr)(payload))
+                if ((0, result_1.isErr)(payload)) {
+                    recordRuntimeError(payload.error, {
+                        surface: "feed",
+                        direction: "server",
+                        operation: feed,
+                        phase: "event_encoding",
+                    });
                     return payload;
+                }
                 if (!msg.reply) {
-                    return (0, result_1.err)(new index_js_1.UnexpectedError({
+                    const error = new index_js_1.UnexpectedError({
                         context: { feed, reason: "missing_reply" },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "feed",
+                        direction: "server",
+                        operation: feed,
+                        phase: "event_publish",
+                    });
+                    return (0, result_1.err)(error);
+                }
+                try {
+                    this.nats.publish(msg.reply, payload);
+                    await this.nats.flush();
+                }
+                catch (cause) {
+                    const error = new index_js_1.UnexpectedError({
+                        cause,
+                        context: { feed },
+                    });
+                    recordRuntimeError(error, {
+                        surface: "feed",
+                        direction: "server",
+                        operation: feed,
+                        phase: "event_publish",
+                    });
+                    return (0, result_1.err)(error);
                 }
-                this.nats.publish(msg.reply, payload);
-                await this.nats.flush();
                 return (0, result_1.ok)(undefined);
             })()),
         });
@@ -1640,14 +1882,21 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             ? handlerResult.take()
             : handlerResult;
         if ((0, result_1.isErr)(handlerOutcome)) {
-            return (0, result_1.err)(annotateHandlerBoundaryError(handlerOutcome.error, {
+            const error = annotateHandlerBoundaryError(handlerOutcome.error, {
                 feed,
                 requestId: msg.headers?.get("request-id"),
                 service: this.name,
                 contractId: this.contractId,
                 contractDigest: this.contractDigest,
                 traceId: traceIdFromTraceparent(msg.headers?.get("traceparent")),
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "feed",
+                direction: "server",
+                operation: feed,
+                phase: "handler_result",
+            });
+            return (0, result_1.err)(error);
         }
         return (0, result_1.ok)(undefined);
     }
@@ -1699,31 +1948,43 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
 }, _Trellis_processRPCMessage = async function _Trellis_processRPCMessage(method, ctx, msg, fn, handlerTrellis) {
     __classPrivateFieldGet(this, _Trellis_log, "f").debug({ method: String(method), subject: msg.subject }, "Processing RPC message");
     // Extract trace context from incoming NATS headers
-    const parentContext = (0, tracing_js_1.extractTraceContext)((0, tracing_js_1.createNatsHeaderCarrier)({
+    const parentContext = (0, mod_js_2.extractTraceContext)((0, mod_js_2.createNatsHeaderCarrier)({
         get: (k) => msg.headers?.get(k) ?? undefined,
         set: () => { }, // Server doesn't need to set headers on incoming messages
     }));
     // Start a server span for this RPC handler
-    const span = (0, tracing_js_1.startServerSpan)(method, msg.subject, parentContext);
+    const span = (0, mod_js_2.startServerSpan)(method, msg.subject, parentContext);
     const incomingTraceId = traceIdFromTraceparent(msg.headers?.get("traceparent"));
     // Execute the handler within the span's context
-    return (0, tracing_js_1.withSpanAsync)(span, async () => {
+    return (0, mod_js_2.withSpanAsync)(span, async () => {
         const execute = async () => {
             const jsonData = safeJson(msg).take();
             if ((0, result_1.isErr)(jsonData)) {
                 __classPrivateFieldGet(this, _Trellis_log, "f").warn({ method, error: jsonData.error.message }, "Failed to parse JSON");
                 span.setStatus({
-                    code: tracing_js_1.SpanStatusCode.ERROR,
+                    code: mod_js_2.SpanStatusCode.ERROR,
                     message: "Failed to parse JSON",
                 });
+                recordRuntimeError(jsonData.error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "parse",
+                });
                 return jsonData;
             }
             const parsedInput = parseRuntimeSchema(ctx.input, jsonData).take();
             if ((0, result_1.isErr)(parsedInput)) {
                 span.setStatus({
-                    code: tracing_js_1.SpanStatusCode.ERROR,
+                    code: mod_js_2.SpanStatusCode.ERROR,
                     message: "Input validation failed",
                 });
+                recordRuntimeError(parsedInput.error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "input_validation",
+                });
                 return parsedInput;
             }
             let caller;
@@ -1746,22 +2007,43 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 if (!sessionKey) {
                     __classPrivateFieldGet(this, _Trellis_log, "f").warn({ method }, "Missing session-key header");
                     span.setStatus({
-                        code: tracing_js_1.SpanStatusCode.ERROR,
+                        code: mod_js_2.SpanStatusCode.ERROR,
                         message: "Missing session-key",
                     });
-                    return (0, result_1.err)(new index_js_1.AuthError({ reason: "missing_session_key" }));
+                    const error = new index_js_1.AuthError({ reason: "missing_session_key" });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return (0, result_1.err)(error);
                 }
                 if (!proof) {
                     __classPrivateFieldGet(this, _Trellis_log, "f").warn({ method }, "Missing proof in request");
                     span.setStatus({
-                        code: tracing_js_1.SpanStatusCode.ERROR,
+                        code: mod_js_2.SpanStatusCode.ERROR,
                         message: "Missing proof",
                     });
-                    return (0, result_1.err)(new index_js_1.AuthError({ reason: "missing_proof" }));
+                    const error = new index_js_1.AuthError({ reason: "missing_proof" });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return (0, result_1.err)(error);
                 }
                 const iat = Number(iatHeader);
                 if (!Number.isSafeInteger(iat) || !requestId) {
-                    return (0, result_1.err)(new index_js_1.AuthError({ reason: "invalid_signature" }));
+                    const error = new index_js_1.AuthError({ reason: "invalid_signature" });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return (0, result_1.err)(error);
                 }
                 // Verify proof signature locally using the raw request bytes we received.
                 const payloadBytes = msg.data ?? new Uint8Array();
@@ -1777,13 +2059,20 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     verifyResult.take() === true;
                 if (!signatureOk) {
                     span.setStatus({
-                        code: tracing_js_1.SpanStatusCode.ERROR,
+                        code: mod_js_2.SpanStatusCode.ERROR,
                         message: "Invalid signature",
                     });
-                    return (0, result_1.err)(new index_js_1.AuthError({
+                    const error = new index_js_1.AuthError({
                         reason: "invalid_signature",
                         context: { sessionKey },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return (0, result_1.err)(error);
                 }
                 let auth;
                 for (let attempt = 0; attempt < DEFAULT_AUTH_VALIDATE_SESSION_RETRY_ATTEMPTS; attempt++) {
@@ -1809,9 +2098,16 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     await sleep(DEFAULT_AUTH_VALIDATE_SESSION_RETRY_MS * (attempt + 1));
                 }
                 if (!auth) {
-                    return (0, result_1.err)(new index_js_1.UnexpectedError({
+                    const error = new index_js_1.UnexpectedError({
                         context: { reason: "missing_auth_validate_result" },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return (0, result_1.err)(error);
                 }
                 if (auth instanceof Error) {
                     __classPrivateFieldGet(this, _Trellis_log, "f").warn({
@@ -1823,37 +2119,64 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                             : undefined,
                     }, "Auth.Requests.Validate failed");
                     span.setStatus({
-                        code: tracing_js_1.SpanStatusCode.ERROR,
+                        code: mod_js_2.SpanStatusCode.ERROR,
                         message: "Auth.Requests.Validate failed",
                     });
                     if (auth instanceof result_1.BaseError) {
+                        recordRuntimeError(auth, {
+                            surface: "rpc",
+                            direction: "server",
+                            operation: String(method),
+                            phase: "auth",
+                        });
                         return (0, result_1.err)(auth);
                     }
-                    return (0, result_1.err)(new index_js_1.UnexpectedError({ cause: auth }));
+                    const error = new index_js_1.UnexpectedError({ cause: auth });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return (0, result_1.err)(error);
                 }
                 if (!auth.allowed) {
                     span.setStatus({
-                        code: tracing_js_1.SpanStatusCode.ERROR,
+                        code: mod_js_2.SpanStatusCode.ERROR,
                         message: "Insufficient permissions",
                     });
-                    return (0, result_1.err)(new index_js_1.AuthError({
+                    const error = new index_js_1.AuthError({
                         reason: "insufficient_permissions",
                         context: {
                             requiredCapabilities: ctx.callerCapabilities,
                             userCapabilities: auth.caller.capabilities,
                         },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return (0, result_1.err)(error);
                 }
                 if (typeof msg.reply !== "string" ||
                     !msg.reply.startsWith(`${auth.inboxPrefix}.`)) {
                     span.setStatus({
-                        code: tracing_js_1.SpanStatusCode.ERROR,
+                        code: mod_js_2.SpanStatusCode.ERROR,
                         message: "Reply subject mismatch",
                     });
-                    return (0, result_1.err)(new index_js_1.AuthError({
+                    const error = new index_js_1.AuthError({
                         reason: "reply_subject_mismatch",
                         context: { expected: auth.inboxPrefix, actual: msg.reply },
-                    }));
+                    });
+                    recordRuntimeError(error, {
+                        surface: "rpc",
+                        direction: "server",
+                        operation: String(method),
+                        phase: "auth",
+                    });
+                    return (0, result_1.err)(error);
                 }
                 caller = auth.caller;
             }
@@ -1897,10 +2220,16 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         : error.cause,
                 }, "Handler threw unexpectedly.");
                 span.setStatus({
-                    code: tracing_js_1.SpanStatusCode.ERROR,
+                    code: mod_js_2.SpanStatusCode.ERROR,
                     message: error.message,
                 });
                 span.recordException(error);
+                recordRuntimeError(error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "handler_throw",
+                });
                 return (0, result_1.err)(error);
             }
             const handlerResult = handlerResultWrapped.take();
@@ -1923,20 +2252,32 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         : error.cause,
                 }, "Handler returned error.");
                 span.setStatus({
-                    code: tracing_js_1.SpanStatusCode.ERROR,
+                    code: mod_js_2.SpanStatusCode.ERROR,
                     message: error.message,
                 });
+                recordRuntimeError(error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "handler_result",
+                });
                 return (0, result_1.err)(error);
             }
             const encoded = (0, codec_js_1.encodeSchema)(ctx.output, handlerOutcome).take();
             if ((0, result_1.isErr)(encoded)) {
                 span.setStatus({
-                    code: tracing_js_1.SpanStatusCode.ERROR,
+                    code: mod_js_2.SpanStatusCode.ERROR,
                     message: "Output encoding failed",
                 });
+                recordRuntimeError(encoded.error, {
+                    surface: "rpc",
+                    direction: "server",
+                    operation: String(method),
+                    phase: "output_encoding",
+                });
                 return encoded;
             }
-            span.setStatus({ code: tracing_js_1.SpanStatusCode.OK });
+            span.setStatus({ code: mod_js_2.SpanStatusCode.OK });
             return (0, result_1.ok)(encoded);
         };
         const result = await execute();
@@ -2010,6 +2351,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             const m = parsedEvent.take();
             if ((0, result_1.isErr)(m)) {
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({ error: m.error }, "Event validation failed");
+                recordRuntimeError(m.error, {
+                    surface: "event",
+                    direction: "consumer",
+                    operation: String(event),
+                    phase: "input_validation",
+                });
                 continue;
             }
             const handlerResult = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_invokeEventHandler).call(this, {
@@ -2021,6 +2368,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             });
             const handlerValue = handlerResult.take();
             if ((0, result_1.isErr)(handlerValue)) {
+                recordRuntimeError(handlerValue.error, {
+                    surface: "event",
+                    direction: "consumer",
+                    operation: String(event),
+                    phase: "handler_result",
+                });
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({
                     error: handlerValue.error.toSerializable(),
                     event,
@@ -2220,6 +2573,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 const eventPayload = parsedEvent.take();
                 if ((0, result_1.isErr)(eventPayload)) {
                     __classPrivateFieldGet(this, _Trellis_log, "f").error({ error: eventPayload.error }, "Event validation failed");
+                    recordRuntimeError(eventPayload.error, {
+                        surface: "event",
+                        direction: "consumer",
+                        operation: String(registration.event),
+                        phase: "input_validation",
+                    });
                     msg.term();
                     failed = true;
                     break;
@@ -2234,6 +2593,12 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 });
                 const handlerValue = handlerResult.take();
                 if ((0, result_1.isErr)(handlerValue)) {
+                    recordRuntimeError(handlerValue.error, {
+                        surface: "event",
+                        direction: "consumer",
+                        operation: String(registration.event),
+                        phase: "handler_result",
+                    });
                     __classPrivateFieldGet(this, _Trellis_log, "f").error({
                         error: handlerValue.error.toSerializable(),
                         event: registration.event,
@@ -2309,8 +2674,8 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
     }));
 }, _Trellis_requestJson = function _Trellis_requestJson(subject, body) {
     return result_1.AsyncResult.from((async () => {
-        const span = (0, tracing_js_1.startClientSpan)(subject, subject);
-        return await (0, tracing_js_1.withSpanAsync)(span, async () => {
+        const span = (0, mod_js_2.startClientSpan)(subject, subject);
+        return await (0, mod_js_2.withSpanAsync)(span, async () => {
             try {
                 const payload = JSON.stringify(body);
                 const authHeaders = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_createProof).call(this, subject, payload);
@@ -2319,7 +2684,7 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 headers.set("proof", authHeaders.proof);
                 headers.set("iat", String(authHeaders.iat));
                 headers.set("request-id", authHeaders.requestId);
-                (0, tracing_js_1.injectTraceContext)((0, tracing_js_1.createNatsHeaderCarrier)(headers), span);
+                (0, mod_js_2.injectTraceContext)((0, mod_js_2.createNatsHeaderCarrier)(headers), span);
                 const response = (await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_requestMessageWithRetry).call(this, {
                     subject,
                     payload,
@@ -2328,9 +2693,15 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 })).take();
                 if ((0, result_1.isErr)(response)) {
                     span.setStatus({
-                        code: tracing_js_1.SpanStatusCode.ERROR,
+                        code: mod_js_2.SpanStatusCode.ERROR,
                         message: response.error.message,
                     });
+                    recordRuntimeError(response.error, {
+                        surface: "operation",
+                        direction: "client",
+                        operation: "requestJson",
+                        phase: "request_send",
+                    });
                     return response;
                 }
                 const json = safeJson(response).take();
@@ -2343,21 +2714,33 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                         context: { subject },
                     });
                     span.setStatus({
-                        code: tracing_js_1.SpanStatusCode.ERROR,
+                        code: mod_js_2.SpanStatusCode.ERROR,
                         message: error.message,
                     });
+                    recordRuntimeError(error, {
+                        surface: "operation",
+                        direction: "client",
+                        operation: "requestJson",
+                        phase: "response_decoding",
+                    });
                     return (0, result_1.err)(error);
                 }
-                span.setStatus({ code: tracing_js_1.SpanStatusCode.OK });
+                span.setStatus({ code: mod_js_2.SpanStatusCode.OK });
                 return (0, result_1.ok)(json);
             }
             catch (cause) {
                 const error = new index_js_1.UnexpectedError({ cause });
                 span.setStatus({
-                    code: tracing_js_1.SpanStatusCode.ERROR,
+                    code: mod_js_2.SpanStatusCode.ERROR,
                     message: error.message,
                 });
                 span.recordException(error);
+                recordRuntimeError(error, {
+                    surface: "operation",
+                    direction: "client",
+                    operation: "requestJson",
+                    phase: "unexpected",
+                });
                 return (0, result_1.err)(error);
             }
             finally {
@@ -2385,35 +2768,56 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
         }
         catch (cause) {
             sub.unsubscribe();
-            return (0, result_1.err)(createTransportError({
+            const error = createTransportError({
                 code: "trellis.watch.failed",
                 message: "Trellis could not start the operation watch.",
                 hint: "Retry watching the operation. If it keeps failing, reconnect to Trellis and try again.",
                 cause,
                 context: { subject },
-            }));
+            });
+            recordRuntimeError(error, {
+                surface: "operation",
+                direction: "client",
+                operation: "watchJson",
+                phase: "request_send",
+            });
+            return (0, result_1.err)(error);
         }
         return (0, result_1.ok)((async function* () {
             try {
                 for await (const msg of sub) {
                     if (msg.headers?.get("status") === "error") {
-                        yield (0, result_1.err)(createTransportError({
+                        const error = createTransportError({
                             code: "trellis.watch.failed",
                             message: "Trellis stopped the operation watch.",
                             hint: "Retry watching the operation. If it keeps happening, reconnect to Trellis and try again.",
                             context: { subject, frame: msg.string() },
-                        }));
+                        });
+                        recordRuntimeError(error, {
+                            surface: "operation",
+                            direction: "client",
+                            operation: "watchJson",
+                            phase: "remote_error",
+                        });
+                        yield (0, result_1.err)(error);
                         continue;
                     }
                     const json = safeJson(msg).take();
                     if ((0, result_1.isErr)(json)) {
-                        yield (0, result_1.err)(createTransportError({
+                        const error = createTransportError({
                             code: "trellis.watch.invalid_response",
                             message: "Trellis returned an invalid watch update.",
                             hint: "Retry watching the operation. If it keeps happening, reconnect to Trellis and try again.",
                             cause: json.error.cause,
                             context: { subject },
-                        }));
+                        });
+                        recordRuntimeError(error, {
+                            surface: "operation",
+                            direction: "client",
+                            operation: "watchJson",
+                            phase: "response_decoding",
+                        });
+                        yield (0, result_1.err)(error);
                         continue;
                     }
                     yield (0, result_1.ok)(json);