@qlever-llc/trellis 0.10.10 → 0.10.11

package/script/trellis.js

@@ -10,7 +10,7 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
     if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
     return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
 };
-var _Trellis_instances, _Trellis_log, _Trellis_tasks, _Trellis_hasExplicitApi, _Trellis_noResponderMaxRetries, _Trellis_noResponderRetryMs, _Trellis_onSessionNotFound, _Trellis_operationStore, _Trellis_eventConsumers, _Trellis_durableEventLoops, _Trellis_createStateFacade, _Trellis_createRpcFacade, _Trellis_createRpcHandleFacade, _Trellis_createEventFacade, _Trellis_createEventPublishFacade, _Trellis_createFeedFacade, _Trellis_createHandlerTrellis, _Trellis_createOperationFacade, _Trellis_unknownApiError, _Trellis_requestBuiltRpcUnknown, _Trellis_requestBuiltRpc, _Trellis_handleBrowserAuthRequired, _Trellis_authenticateFeedRequest, _Trellis_subscribeFeed, _Trellis_handleFeed, _Trellis_processFeedMessage, _Trellis_handleRPC, _Trellis_processRPCMessage, _Trellis_respondWithPayload, _Trellis_respondWithError, _Trellis_startEphemeralEvent, _Trellis_resolveEventConsumerGroup, _Trellis_registerDurableEventHandler, _Trellis_startDurableEventConsumer, _Trellis_durableEventConsumerGroupReady, _Trellis_runDurableEventConsumer, _Trellis_handleDurableEvent, _Trellis_handleDurableEventConsumer, _Trellis_parseEventMessage, _Trellis_escapeSubjectToken, _Trellis_currentIat, _Trellis_createProof, _Trellis_requestMessageWithRetry, _Trellis_requestJson, _Trellis_watchJson;
+var _Trellis_instances, _Trellis_log, _Trellis_tasks, _Trellis_hasExplicitApi, _Trellis_noResponderMaxRetries, _Trellis_noResponderRetryMs, _Trellis_onSessionNotFound, _Trellis_operationStore, _Trellis_eventConsumers, _Trellis_durableEventLoops, _Trellis_createStateFacade, _Trellis_createRpcFacade, _Trellis_createRpcHandleFacade, _Trellis_createEventFacade, _Trellis_createEventPublishFacade, _Trellis_createFeedFacade, _Trellis_createHandlerTrellis, _Trellis_createOperationFacade, _Trellis_unknownApiError, _Trellis_requestBuiltRpcUnknown, _Trellis_requestBuiltRpc, _Trellis_handleBrowserAuthRequired, _Trellis_authenticateFeedRequest, _Trellis_subscribeFeed, _Trellis_handleFeed, _Trellis_processFeedMessage, _Trellis_handleRPC, _Trellis_processRPCMessage, _Trellis_respondWithPayload, _Trellis_respondWithError, _Trellis_startEphemeralEvent, _Trellis_invokeEventHandler, _Trellis_resolveEventConsumerGroup, _Trellis_registerDurableEventHandler, _Trellis_startDurableEventConsumer, _Trellis_durableEventConsumerGroupReady, _Trellis_runDurableEventConsumer, _Trellis_handleDurableEvent, _Trellis_handleDurableEventConsumer, _Trellis_parseEventMessage, _Trellis_escapeSubjectToken, _Trellis_currentIat, _Trellis_createProof, _Trellis_requestMessageWithRetry, _Trellis_requestJson, _Trellis_watchJson;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Trellis = exports.DurableOperationRecordSchema = void 0;
 exports.safeJson = safeJson;
@@ -21,6 +21,7 @@ exports.sha256 = sha256;
 exports.buildProofInput = buildProofInput;
 exports.isOperationDeferred = isOperationDeferred;
 exports.isResultLike = isResultLike;
+exports.annotateHandlerBoundaryError = annotateHandlerBoundaryError;
 exports.buildRuntimeOperationSnapshot = buildRuntimeOperationSnapshot;
 exports.isTerminalRuntimeOperationSnapshot = isTerminalRuntimeOperationSnapshot;
 exports.createTrellisInternal = createTrellisInternal;
@@ -204,6 +205,21 @@ function isOperationDeferred(value) {
 function isResultLike(value) {
     return value instanceof result_1.Result;
 }
+function compactHandlerErrorContext(context) {
+    return Object.fromEntries(Object.entries(context).filter(([key, value]) => key !== "traceId" && value !== undefined));
+}
+function sanitizeHandlerErrorContext(error) {
+    delete error.getContext().subject;
+}
+function annotateHandlerBoundaryError(cause, context) {
+    const error = cause instanceof result_1.BaseError && !(cause instanceof RemoteError_js_1.RemoteError)
+        ? cause
+        : new index_js_1.UnexpectedError({ cause });
+    sanitizeHandlerErrorContext(error);
+    error.withContext(compactHandlerErrorContext(context));
+    error.withTraceId(context.traceId);
+    return error;
+}
 const DurableOperationSignalSchema = typebox_1.Type.Object({
     operationId: typebox_1.Type.String(),
     sequence: typebox_1.Type.Number(),
@@ -234,8 +250,11 @@ const DurableOperationSnapshotSchema = typebox_1.Type.Object({
     })),
     output: typebox_1.Type.Optional(typebox_1.Type.Any()),
     error: typebox_1.Type.Optional(typebox_1.Type.Object({
+        id: typebox_1.Type.Optional(typebox_1.Type.String()),
         type: typebox_1.Type.String(),
         message: typebox_1.Type.String(),
+        context: typebox_1.Type.Optional(typebox_1.Type.Record(typebox_1.Type.String(), typebox_1.Type.Unknown())),
+        traceId: typebox_1.Type.Optional(typebox_1.Type.String()),
     })),
 });
 exports.DurableOperationRecordSchema = typebox_1.Type.Object({
@@ -741,6 +760,18 @@ class Trellis {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "contractId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "contractDigest", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "nats", {
             enumerable: true,
             configurable: true,
@@ -783,6 +814,8 @@ class Trellis {
         __classPrivateFieldSet(this, _Trellis_log, (opts?.log ?? globals_js_1.logger).child({ lib: "trellis" }), "f");
         this.timeout = opts?.timeout ?? 3000;
         this.stream = opts?.stream ?? "trellis";
+        this.contractId = opts?.contractId;
+        this.contractDigest = opts?.contractDigest;
         __classPrivateFieldSet(this, _Trellis_hasExplicitApi, api !== undefined, "f");
         __classPrivateFieldSet(this, _Trellis_noResponderMaxRetries, opts?.noResponderRetry?.maxAttempts ??
             DEFAULT_NO_RESPONDER_MAX_RETRIES, "f");
@@ -1543,9 +1576,14 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     }
                 }
                 catch (cause) {
-                    const error = cause instanceof result_1.BaseError
-                        ? cause
-                        : new index_js_1.UnexpectedError({ cause });
+                    const error = annotateHandlerBoundaryError(cause, {
+                        feed,
+                        requestId: msg.headers?.get("request-id"),
+                        service: this.name,
+                        contractId: this.contractId,
+                        contractDigest: this.contractDigest,
+                        traceId: traceIdFromTraceparent(msg.headers?.get("traceparent")),
+                    });
                     __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_respondWithError).call(this, msg, error);
                 }
             })();
@@ -1580,7 +1618,7 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
     await this.nats.flush();
     const controller = new AbortController();
     try {
-        await handler({
+        const handlerResult = await handler({
             input: parsed,
             caller: callerValue,
             signal: controller.signal,
@@ -1598,6 +1636,19 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 return (0, result_1.ok)(undefined);
             })()),
         });
+        const handlerOutcome = isResultLike(handlerResult)
+            ? handlerResult.take()
+            : handlerResult;
+        if ((0, result_1.isErr)(handlerOutcome)) {
+            return (0, result_1.err)(annotateHandlerBoundaryError(handlerOutcome.error, {
+                feed,
+                requestId: msg.headers?.get("request-id"),
+                service: this.name,
+                contractId: this.contractId,
+                contractDigest: this.contractDigest,
+                traceId: traceIdFromTraceparent(msg.headers?.get("traceparent")),
+            }));
+        }
         return (0, result_1.ok)(undefined);
     }
     finally {
@@ -1830,7 +1881,14 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 client: handlerTrellis,
             })));
             if (handlerResultWrapped.isErr()) {
-                const error = handlerResultWrapped.error.withContext({ method });
+                const error = annotateHandlerBoundaryError(handlerResultWrapped.error, {
+                    method: String(method),
+                    requestId: msg.headers?.get("request-id"),
+                    service: this.name,
+                    contractId: this.contractId,
+                    contractDigest: this.contractDigest,
+                    traceId: activeTraceId(span) ?? incomingTraceId,
+                });
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({
                     method,
                     error: error.message,
@@ -1848,11 +1906,14 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
             const handlerResult = handlerResultWrapped.take();
             const handlerOutcome = handlerResult.take();
             if ((0, result_1.isErr)(handlerOutcome)) {
-                const handlerError = handlerOutcome.error;
-                const error = handlerError instanceof result_1.BaseError &&
-                    !(handlerError instanceof RemoteError_js_1.RemoteError)
-                    ? handlerError
-                    : new index_js_1.UnexpectedError({ cause: handlerError });
+                const error = annotateHandlerBoundaryError(handlerOutcome.error, {
+                    method: String(method),
+                    requestId: msg.headers?.get("request-id"),
+                    service: this.name,
+                    contractId: this.contractId,
+                    contractDigest: this.contractDigest,
+                    traceId: activeTraceId(span) ?? incomingTraceId,
+                });
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({
                     method,
                     error: error.message,
@@ -1951,15 +2012,17 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({ error: m.error }, "Event validation failed");
                 continue;
             }
-            const handlerResult = await result_1.AsyncResult.lift(fn(m, createEventListenerContext({
+            const handlerResult = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_invokeEventHandler).call(this, {
+                event,
                 payload: m,
-                subject: msg.subject,
                 mode: "ephemeral",
                 message: msg,
-            })));
-            if (handlerResult.isErr()) {
+                fn,
+            });
+            const handlerValue = handlerResult.take();
+            if ((0, result_1.isErr)(handlerValue)) {
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({
-                    error: handlerResult.error.toSerializable(),
+                    error: handlerValue.error.toSerializable(),
                     event,
                     subject: msg.subject,
                 }, "Event handler failed");
@@ -1968,6 +2031,31 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
     });
     __classPrivateFieldGet(this, _Trellis_tasks, "f").add(`event:${event}:${(0, ulid_1.ulid)()}`, task);
     return (0, result_1.ok)(undefined);
+}, _Trellis_invokeEventHandler = async function _Trellis_invokeEventHandler(args) {
+    const annotation = {
+        event: String(args.event),
+        service: this.name,
+        contractId: this.contractId,
+        contractDigest: this.contractDigest,
+        traceId: traceIdFromTraceparent(args.message.headers?.get("traceparent")),
+    };
+    try {
+        const result = await Promise.resolve(args.fn(args.payload, createEventListenerContext({
+            payload: args.payload,
+            subject: args.message.subject,
+            mode: args.mode,
+            ...(args.group ? { group: args.group } : {}),
+            message: args.message,
+        })));
+        const outcome = isResultLike(result) ? result.take() : result;
+        if ((0, result_1.isErr)(outcome)) {
+            return (0, result_1.err)(annotateHandlerBoundaryError(outcome.error, annotation));
+        }
+        return (0, result_1.ok)(undefined);
+    }
+    catch (cause) {
+        return (0, result_1.err)(annotateHandlerBoundaryError(cause, annotation));
+    }
 }, _Trellis_resolveEventConsumerGroup = function _Trellis_resolveEventConsumerGroup(event, opts) {
     const metadata = __classPrivateFieldGet(this, _Trellis_eventConsumers, "f").metadata;
     const bindings = __classPrivateFieldGet(this, _Trellis_eventConsumers, "f").bindings ?? {};
@@ -2093,15 +2181,17 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                 msg.term();
                 continue;
             }
-            const handlerResult = await result_1.AsyncResult.lift(fn(m, createEventListenerContext({
+            const handlerResult = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_invokeEventHandler).call(this, {
+                event,
                 payload: m,
-                subject: msg.subject,
                 mode: "durable",
                 message: msg,
-            })));
-            if (handlerResult.isErr()) {
+                fn,
+            });
+            const handlerValue = handlerResult.take();
+            if ((0, result_1.isErr)(handlerValue)) {
                 __classPrivateFieldGet(this, _Trellis_log, "f").error({
-                    error: handlerResult.error.toSerializable(),
+                    error: handlerValue.error.toSerializable(),
                     event,
                     subject: msg.subject,
                 }, "Event handler failed");
@@ -2134,16 +2224,18 @@ _Trellis_log = new WeakMap(), _Trellis_tasks = new WeakMap(), _Trellis_hasExplic
                     failed = true;
                     break;
                 }
-                const handlerResult = await result_1.AsyncResult.lift(registration.fn(eventPayload, createEventListenerContext({
+                const handlerResult = await __classPrivateFieldGet(this, _Trellis_instances, "m", _Trellis_invokeEventHandler).call(this, {
+                    event: registration.event,
                     payload: eventPayload,
-                    subject: msg.subject,
                     mode: "durable",
                     group,
                     message: msg,
-                })));
-                if (handlerResult.isErr()) {
+                    fn: registration.fn,
+                });
+                const handlerValue = handlerResult.take();
+                if ((0, result_1.isErr)(handlerValue)) {
                     __classPrivateFieldGet(this, _Trellis_log, "f").error({
-                        error: handlerResult.error.toSerializable(),
+                        error: handlerValue.error.toSerializable(),
                         event: registration.event,
                         subject: msg.subject,
                     }, "Event handler failed");

package/src/client.ts

@@ -50,6 +50,8 @@ type ClientContractManifest = {
 };
 
 type ClientContractShape = {
+  CONTRACT_ID?: string;
+  CONTRACT_DIGEST?: string;
   CONTRACT: ClientContractManifest;
   API: {
     owned?: TrellisAPI;
@@ -120,6 +122,8 @@ export function createClient<
       noResponderRetry: opts?.noResponderRetry,
       api,
       state: contract[CONTRACT_STATE_METADATA],
+      contractId: contract.CONTRACT_ID,
+      contractDigest: contract.CONTRACT_DIGEST,
     },
   );
 }

package/src/client_connect.ts

@@ -224,13 +224,12 @@ const ClientTransportsSchema = Type.Object({
 type ClientConnectDeps = {
   loadTransport(): Promise<RuntimeTransport>;
   now(): number;
-  setInterval?: (
-    handler: () => void,
-    ms: number,
-  ) => ReturnType<typeof globalThis.setInterval>;
-  clearInterval?: (id: ReturnType<typeof globalThis.setInterval>) => void;
+  setInterval?: (handler: () => void, ms: number) => IntervalHandle;
+  clearInterval?: (id: IntervalHandle) => void;
 };
 
+type IntervalHandle = ReturnType<typeof globalThis.setInterval> | number;
+
 const ClientBootstrapReadySchema = Type.Object({
   status: Type.Literal("ready"),
   serverNow: Type.Integer(),
@@ -800,11 +799,14 @@ async function createRuntimeUserAuthenticator(args: {
     ((
       handler: () => void,
       ms: number,
-    ): ReturnType<typeof globalThis.setInterval> =>
-      globalThis.setInterval(handler, ms));
+    ): IntervalHandle => globalThis.setInterval(handler, ms));
   const clearRefreshInterval = args.deps.clearInterval ??
-    ((id: ReturnType<typeof globalThis.setInterval>) =>
-      globalThis.clearInterval(id));
+    ((id: IntervalHandle) => {
+      const clearIntervalFn = globalThis.clearInterval as (
+        id: IntervalHandle,
+      ) => void;
+      clearIntervalFn(id);
+    });
   const refreshIntervalId = setRefreshInterval(() => {
     void refresh();
   }, 10_000);

package/src/server/health.ts

@@ -104,6 +104,25 @@ function summarizeHealthChecks(
   return `${failedCount} check${failedCount === 1 ? "" : "s"} failing`;
 }
 
+function annotateServiceHealthCheck(
+  result: HealthCheckResult,
+  metadata: { service: string; contractId: string; contractDigest: string },
+): HealthCheckResult {
+  if (result.status !== "failed") {
+    return result;
+  }
+
+  return {
+    ...result,
+    info: {
+      ...(result.info ?? {}),
+      service: metadata.service,
+      contractId: metadata.contractId,
+      contractDigest: metadata.contractDigest,
+    },
+  };
+}
+
 function normalizeLegacyHealthCheck(
   check: HealthCheckFn,
 ): ServiceHealthCheckFn {
@@ -114,6 +133,10 @@ function normalizeLegacyHealthCheck(
         status: "failed",
         error: result.error.message,
         summary: result.error.message,
+        info: {
+          errorType: result.error.name,
+          errorId: result.error.id,
+        },
       };
     }
 
@@ -206,6 +229,9 @@ export async function runServiceHealthCheck(
       status: "failed",
       error: message,
       summary: message,
+      info: {
+        errorType: error instanceof Error ? error.name : typeof error,
+      },
       latencyMs,
     };
   }
@@ -350,16 +376,28 @@ export class ServiceHealth {
   }
 
   async checks(): Promise<HealthCheckResult[]> {
-    return await Promise.all(
+    const results = await Promise.all(
       Array.from(this.#checks.entries()).map(([name, check]) =>
         runServiceHealthCheck(name, check)
       ),
     );
+    return results.map((result) =>
+      annotateServiceHealthCheck(result, {
+        service: this.serviceName,
+        contractId: this.contractId,
+        contractDigest: this.contractDigest,
+      })
+    );
   }
 
   async response(): Promise<HealthResponse> {
-    const checks = Object.fromEntries(this.#checks.entries());
-    return await runAllServiceHealthChecks(this.serviceName, checks);
+    const results = await this.checks();
+    return {
+      status: summarizeHealthStatus(results),
+      service: this.serviceName,
+      timestamp: new Date().toISOString(),
+      checks: results,
+    };
   }
 
   async heartbeat(): Promise<Omit<HealthHeartbeat, "header">> {

package/src/server/service.ts

@@ -77,7 +77,10 @@ import type {
   RpcHandlerContext,
   RpcHandlerErrorOf,
 } from "../trellis.js";
-import { createTrellisInternal } from "../trellis.js";
+import {
+  annotateHandlerBoundaryError,
+  createTrellisInternal,
+} from "../trellis.js";
 import type {
   NatsConnectFn,
   NatsConnectOpts,
@@ -1818,6 +1821,8 @@ export async function createConnectedService<
       stream: args.server.stream,
       noResponderRetry: args.server.noResponderRetry,
       api: runtimeApi,
+      contractId: args.contractId,
+      contractDigest: args.contractDigest,
       connection,
       transferSupport: {
         openOperationTransfer: (transferArgs) =>
@@ -1837,6 +1842,8 @@ export async function createConnectedService<
       stream: args.server.stream,
       noResponderRetry: args.server.noResponderRetry,
       api: runtimeApi,
+      contractId: args.contractId,
+      contractDigest: args.contractDigest,
       eventConsumers: {
         metadata: args.contractEventConsumers,
         bindings: args.bindings.eventConsumers,
@@ -2009,6 +2016,14 @@ function toUnexpectedError(cause: unknown): UnexpectedError {
     : new UnexpectedError({ cause });
 }
 
+function serializeJobHandlerError(error: BaseError): string {
+  try {
+    return JSON.stringify(error.toSerializable());
+  } catch {
+    return error.message;
+  }
+}
+
 function okVoid(): Result<void, never> {
   return Result.ok(undefined);
 }
@@ -2403,6 +2418,8 @@ function createJobsFacade<
   TKv extends ContractKvMetadata = ContractKvMetadata,
 >(args: {
   serviceName: string;
+  contractId?: string;
+  contractDigest?: string;
   nc: NatsConnection;
   contractJobs: TJobs;
   client: Trellis<TTrellisApi, TKv, TJobs>;
@@ -2566,9 +2583,35 @@ function createJobsFacade<
                   },
                 );
 
-                const handled = (await handler(publicJob)).take();
+                const jobErrorContext = {
+                  jobType: queueType,
+                  requestId: job.context().requestId,
+                  service: args.serviceName,
+                  contractId: args.contractId,
+                  contractDigest: args.contractDigest,
+                  traceId: job.context().traceId,
+                };
+
+                let handled: unknown | Result<never, BaseError>;
+                try {
+                  handled = (await handler(publicJob)).take();
+                } catch (cause) {
+                  const annotatedError = annotateHandlerBoundaryError(
+                    cause,
+                    jobErrorContext,
+                  );
+                  throw InternalJobProcessError.failed(
+                    serializeJobHandlerError(annotatedError),
+                  );
+                }
                 if (isErr(handled)) {
-                  throw InternalJobProcessError.failed(handled.error.message);
+                  const annotatedError = annotateHandlerBoundaryError(
+                    handled.error,
+                    jobErrorContext,
+                  );
+                  throw InternalJobProcessError.failed(
+                    serializeJobHandlerError(annotatedError),
+                  );
                 }
                 return handled;
               },
@@ -2756,6 +2799,8 @@ export class TrellisService<
     this.#operationTransfer = operationTransfer;
     const jobs = createJobsFacade<TJobs, TTrellisApi, TKv>({
       serviceName: name,
+      contractId: health.contractId,
+      contractDigest: health.contractDigest,
       nc,
       contractJobs,
       client: handlerTrellis,

package/src/server.ts

@@ -27,6 +27,7 @@ import type { LoggerLike } from "./globals.js";
 import { serverLogger } from "./server_logger.js";
 import {
   type AcceptedOperation,
+  annotateHandlerBoundaryError,
   type AnyTrellisAPI,
   type AuthRequestsValidateResponse,
   base64urlDecode,
@@ -199,6 +200,26 @@ function asOptionalStringRecordPointerValue(
   return ok(Object.fromEntries(entries) as Record<string, string>);
 }
 
+function traceIdFromTraceparent(
+  traceparent: string | undefined,
+): string | undefined {
+  const [version, traceId, parentId, flags, extra] = traceparent?.split("-") ??
+    [];
+  if (
+    extra !== undefined ||
+    !/^[0-9a-f]{2}$/u.test(version ?? "") ||
+    version === "ff" ||
+    !/^[0-9a-f]{32}$/u.test(traceId ?? "") ||
+    traceId === "00000000000000000000000000000000" ||
+    !/^[0-9a-f]{16}$/u.test(parentId ?? "") ||
+    parentId === "0000000000000000" ||
+    !/^[0-9a-f]{2}$/u.test(flags ?? "")
+  ) {
+    return undefined;
+  }
+  return traceId;
+}
+
 export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
   #version?: string;
   #log: LoggerLike;
@@ -243,7 +264,7 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
         }),
       fail: (operationId, error) =>
         this.#applyOperationUpdate(operationId, "failed", {
-          patch: { error: { type: error.name, message: error.message } },
+          patch: { error: error.toSerializable() },
           event: { type: "failed" },
         }),
       cancel: (operationId) =>
@@ -506,7 +527,7 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
         }),
       fail: (error: BaseError) =>
         this.#applyControlledOperationUpdate(runtime, ctx, "failed", {
-          patch: { error: { type: error.name, message: error.message } },
+          patch: { error: error.toSerializable() },
           event: { type: "failed" },
         }),
       cancel: () => {
@@ -1210,7 +1231,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
           runtime.waiters.clear();
         };
 
-        const makeOperation = (runtime: RuntimeOperationRecord) => {
+        const makeOperation = (
+          runtime: RuntimeOperationRecord,
+          context: { requestId?: string; traceId?: string },
+        ) => {
           const ensureActive = () => {
             if (runtime.terminal) {
               return err(
@@ -1291,11 +1315,19 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
               AsyncResult.from((async () => {
                 const active = ensureActive();
                 if (active) return active;
+                const annotatedError = annotateHandlerBoundaryError(error, {
+                  operation: String(operation),
+                  requestId: context.requestId,
+                  service: this.name,
+                  contractId: this.contractId,
+                  contractDigest: this.contractDigest,
+                  traceId: context.traceId,
+                });
                 const snapshot = buildRuntimeOperationSnapshot(
                   runtime,
                   "failed",
                   {
-                    error: { type: error.name, message: error.message },
+                    error: annotatedError.toSerializable(),
                     completedAt: now(),
                   },
                 );
@@ -1636,7 +1668,13 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
             msg.respond(JSON.stringify(accepted));
 
             void (async () => {
-              const op = makeOperation(runtime);
+              const operationContext = {
+                requestId: msg.headers?.get("request-id"),
+                traceId: traceIdFromTraceparent(
+                  msg.headers?.get("traceparent"),
+                ),
+              };
+              const op = makeOperation(runtime, operationContext);
               try {
                 const handlerResult: unknown = await handler(
                   transferSession
@@ -1656,7 +1694,17 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                   ? handlerResult.take()
                   : handlerResult;
                 if (isErr(handlerOutcome)) {
-                  await op.fail(handlerOutcome.error);
+                  await op.fail(annotateHandlerBoundaryError(
+                    handlerOutcome.error,
+                    {
+                      operation: String(operation),
+                      requestId: operationContext.requestId,
+                      service: this.name,
+                      contractId: this.contractId,
+                      contractDigest: this.contractDigest,
+                      traceId: operationContext.traceId,
+                    },
+                  ));
                   return;
                 }
 
@@ -1676,7 +1724,14 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                   await op.complete(handlerOutcome);
                 }
               } catch (cause) {
-                await op.fail(new UnexpectedError({ cause }));
+                await op.fail(annotateHandlerBoundaryError(cause, {
+                  operation: String(operation),
+                  requestId: operationContext.requestId,
+                  service: this.name,
+                  contractId: this.contractId,
+                  contractDigest: this.contractDigest,
+                  traceId: operationContext.traceId,
+                }));
               }
             })();
           }