@qlever-llc/trellis 0.10.10 → 0.10.12

package/src/client.ts

@@ -50,6 +50,8 @@ type ClientContractManifest = {
 };
 
 type ClientContractShape = {
+  CONTRACT_ID?: string;
+  CONTRACT_DIGEST?: string;
   CONTRACT: ClientContractManifest;
   API: {
     owned?: TrellisAPI;
@@ -120,6 +122,8 @@ export function createClient<
       noResponderRetry: opts?.noResponderRetry,
       api,
       state: contract[CONTRACT_STATE_METADATA],
+      contractId: contract.CONTRACT_ID,
+      contractDigest: contract.CONTRACT_DIGEST,
     },
   );
 }

package/src/client_connect.ts

@@ -224,13 +224,12 @@ const ClientTransportsSchema = Type.Object({
 type ClientConnectDeps = {
   loadTransport(): Promise<RuntimeTransport>;
   now(): number;
-  setInterval?: (
-    handler: () => void,
-    ms: number,
-  ) => ReturnType<typeof globalThis.setInterval>;
-  clearInterval?: (id: ReturnType<typeof globalThis.setInterval>) => void;
+  setInterval?: (handler: () => void, ms: number) => IntervalHandle;
+  clearInterval?: (id: IntervalHandle) => void;
 };
 
+type IntervalHandle = ReturnType<typeof globalThis.setInterval> | number;
+
 const ClientBootstrapReadySchema = Type.Object({
   status: Type.Literal("ready"),
   serverNow: Type.Integer(),
@@ -800,11 +799,14 @@ async function createRuntimeUserAuthenticator(args: {
     ((
       handler: () => void,
       ms: number,
-    ): ReturnType<typeof globalThis.setInterval> =>
-      globalThis.setInterval(handler, ms));
+    ): IntervalHandle => globalThis.setInterval(handler, ms));
   const clearRefreshInterval = args.deps.clearInterval ??
-    ((id: ReturnType<typeof globalThis.setInterval>) =>
-      globalThis.clearInterval(id));
+    ((id: IntervalHandle) => {
+      const clearIntervalFn = globalThis.clearInterval as (
+        id: IntervalHandle,
+      ) => void;
+      clearIntervalFn(id);
+    });
   const refreshIntervalId = setRefreshInterval(() => {
     void refresh();
   }, 10_000);

package/src/errors/TrellisError.ts

@@ -1,20 +1,20 @@
 /**
  * Base class for all Trellis-specific errors.
- * Extends BaseError and relies on the traceId getter being configured via initTracing.
+ * Extends BaseError and relies on the traceId getter being configured via initTelemetry.
  */
 import { BaseError, type BaseErrorSchema } from "@qlever-llc/result";
 
 /**
  * Abstract base class for Trellis errors.
- * Trellis errors automatically include traceId when initTracing() has been called
+ * Trellis errors automatically include traceId when initTelemetry() has been called
  * and a span is active in the current context.
  *
- * The traceId integration is configured by the tracing module's initTracing() function,
+ * The traceId integration is configured by the telemetry module's initTelemetry() function,
  * which sets up BaseError.traceIdGetter to retrieve the traceId from the active span.
  */
 export abstract class TrellisError<
   TData extends BaseErrorSchema = BaseErrorSchema,
 > extends BaseError<TData> {
   // TrellisError inherits getTraceId() from BaseError which uses the static traceIdGetter.
-  // The traceIdGetter is configured by initTracing() in the tracing module.
+  // The traceIdGetter is configured by initTelemetry() in the telemetry module.
 }

package/src/server/health.ts

@@ -104,6 +104,25 @@ function summarizeHealthChecks(
   return `${failedCount} check${failedCount === 1 ? "" : "s"} failing`;
 }
 
+function annotateServiceHealthCheck(
+  result: HealthCheckResult,
+  metadata: { service: string; contractId: string; contractDigest: string },
+): HealthCheckResult {
+  if (result.status !== "failed") {
+    return result;
+  }
+
+  return {
+    ...result,
+    info: {
+      ...(result.info ?? {}),
+      service: metadata.service,
+      contractId: metadata.contractId,
+      contractDigest: metadata.contractDigest,
+    },
+  };
+}
+
 function normalizeLegacyHealthCheck(
   check: HealthCheckFn,
 ): ServiceHealthCheckFn {
@@ -114,6 +133,10 @@ function normalizeLegacyHealthCheck(
         status: "failed",
         error: result.error.message,
         summary: result.error.message,
+        info: {
+          errorType: result.error.name,
+          errorId: result.error.id,
+        },
       };
     }
 
@@ -206,6 +229,9 @@ export async function runServiceHealthCheck(
       status: "failed",
       error: message,
       summary: message,
+      info: {
+        errorType: error instanceof Error ? error.name : typeof error,
+      },
       latencyMs,
     };
   }
@@ -350,16 +376,28 @@ export class ServiceHealth {
   }
 
   async checks(): Promise<HealthCheckResult[]> {
-    return await Promise.all(
+    const results = await Promise.all(
       Array.from(this.#checks.entries()).map(([name, check]) =>
         runServiceHealthCheck(name, check)
       ),
     );
+    return results.map((result) =>
+      annotateServiceHealthCheck(result, {
+        service: this.serviceName,
+        contractId: this.contractId,
+        contractDigest: this.contractDigest,
+      })
+    );
   }
 
   async response(): Promise<HealthResponse> {
-    const checks = Object.fromEntries(this.#checks.entries());
-    return await runAllServiceHealthChecks(this.serviceName, checks);
+    const results = await this.checks();
+    return {
+      status: summarizeHealthStatus(results),
+      service: this.serviceName,
+      timestamp: new Date().toISOString(),
+      checks: results,
+    };
   }
 
   async heartbeat(): Promise<Omit<HealthHeartbeat, "header">> {

package/src/server/internal_jobs/job-manager.ts

@@ -4,6 +4,7 @@ import {
   createMapCarrier,
   injectTraceContext,
 } from "../../telemetry/carrier.js";
+import { recordTrellisError } from "../../telemetry/mod.js";
 
 import {
   ActiveJob,
@@ -103,11 +104,22 @@ export class JobManager<TPayload = unknown, TResult = unknown> {
   ): Promise<void> {
     const binding = this.#getQueueBinding(type);
     const headers = headersFromJobContext(event.context);
-    await this.#context.nc.publish(
-      `${binding.publishPrefix}.${jobId}.${event.eventType}`,
-      new TextEncoder().encode(JSON.stringify(event)),
-      { headers },
-    );
+    try {
+      await this.#context.nc.publish(
+        `${binding.publishPrefix}.${jobId}.${event.eventType}`,
+        new TextEncoder().encode(JSON.stringify(event)),
+        { headers },
+      );
+    } catch (error) {
+      recordTrellisError(error, {
+        surface: "job",
+        direction: "worker",
+        operation: type,
+        phase: "publish",
+        messagingSystem: "nats",
+      });
+      throw error;
+    }
   }
 
   async create(
@@ -247,6 +259,12 @@ export class JobManager<TPayload = unknown, TResult = unknown> {
       if (error instanceof JobProcessError) {
         const detail = error.message;
         if (error.kind === "retryable") {
+          recordTrellisError(error, {
+            surface: "job",
+            direction: "worker",
+            operation: job.type,
+            phase: "handler_result",
+          });
           await this.#publishJobEvent(job.type, job.id, {
             jobId: job.id,
             service: job.service,
@@ -262,6 +280,12 @@ export class JobManager<TPayload = unknown, TResult = unknown> {
           return { outcome: "retry", tries, error: detail };
         }
 
+        recordTrellisError(error, {
+          surface: "job",
+          direction: "worker",
+          operation: job.type,
+          phase: "handler_result",
+        });
         await this.#publishJobEvent(job.type, job.id, {
           jobId: job.id,
           service: job.service,
@@ -277,6 +301,12 @@ export class JobManager<TPayload = unknown, TResult = unknown> {
         return { outcome: "failed", tries, error: detail };
       }
 
+      recordTrellisError(error, {
+        surface: "job",
+        direction: "worker",
+        operation: job.type,
+        phase: "runtime",
+      });
       throw error;
     }
   }

package/src/server/runtime.ts

@@ -20,8 +20,12 @@ export type NatsCredsAuthenticatorFn = (creds: Uint8Array) => unknown;
 
 export type ReadFileSyncFn = (path: string) => Uint8Array;
 
+/** Initializes telemetry for a service runtime. */
+export type InitTelemetryFn = (serviceName: string) => void;
+
 export type TrellisServiceRuntimeDeps = {
   connect: NatsConnectFn;
   credsAuthenticator?: NatsCredsAuthenticatorFn;
   readFileSync?: ReadFileSyncFn;
+  initTelemetry?: InitTelemetryFn;
 };

package/src/server/service.ts

@@ -77,7 +77,10 @@ import type {
   RpcHandlerContext,
   RpcHandlerErrorOf,
 } from "../trellis.js";
-import { createTrellisInternal } from "../trellis.js";
+import {
+  annotateHandlerBoundaryError,
+  createTrellisInternal,
+} from "../trellis.js";
 import type {
   NatsConnectFn,
   NatsConnectOpts,
@@ -125,6 +128,7 @@ import {
   observeNatsTrellisConnection,
   type TrellisConnection,
 } from "../connection.js";
+import { initTelemetry } from "../telemetry/init.js";
 
 type ExtraNatsConnectOpts = Omit<
   NatsConnectOpts,
@@ -428,6 +432,7 @@ async function loadDefaultServiceRuntimeDeps(): Promise<
 > {
   const transport = await loadDefaultRuntimeTransport();
   return {
+    initTelemetry,
     connect: (
       { servers, token, authenticator, inboxPrefix, ...extraOptions },
     ) =>
@@ -441,6 +446,12 @@ async function loadDefaultServiceRuntimeDeps(): Promise<
   };
 }
 
+function automaticTelemetryEnabled(
+  telemetry: TrellisServiceConnectTelemetryOpts | undefined,
+): boolean {
+  return telemetry !== false && telemetry?.enabled !== false;
+}
+
 const ServiceBootstrapReadySchema = Type.Object({
   status: Type.Literal("ready"),
   serverNow: Type.Integer(),
@@ -868,9 +879,20 @@ export type TrellisServiceConnectOpts<
   contract: ServiceContract<TOwnedApi, TTrellisApi>;
   name: string;
   sessionKeySeed: string;
+  /**
+   * Controls automatic telemetry initialization for this service connection.
+   * Enabled by default; pass `false` or `{ enabled: false }` to disable it.
+   */
+  telemetry?: TrellisServiceConnectTelemetryOpts;
   server?: TrellisServiceServerOpts;
 };
 
+/** Controls automatic telemetry initialization for `TrellisService.connect()`. */
+export type TrellisServiceConnectTelemetryOpts = false | {
+  /** Whether automatic telemetry initialization is enabled. Defaults to `true`. */
+  enabled?: boolean;
+};
+
 type ServiceKvFacade<TKv extends ContractKvMetadata> = {
   [K in keyof TKv]: TKv[K]["required"] extends false
     ? TypedKV<TKv[K]["schema"]> | undefined
@@ -1757,6 +1779,11 @@ export type TrellisServiceConnectArgs<
   contract: TContract;
   name: string;
   sessionKeySeed: string;
+  /**
+   * Controls automatic telemetry initialization for this service connection.
+   * Enabled by default; pass `false` or `{ enabled: false }` to disable it.
+   */
+  telemetry?: TrellisServiceConnectTelemetryOpts;
   server?: TrellisServiceServerOpts;
 };
 
@@ -1818,6 +1845,8 @@ export async function createConnectedService<
       stream: args.server.stream,
       noResponderRetry: args.server.noResponderRetry,
       api: runtimeApi,
+      contractId: args.contractId,
+      contractDigest: args.contractDigest,
       connection,
       transferSupport: {
         openOperationTransfer: (transferArgs) =>
@@ -1837,6 +1866,8 @@ export async function createConnectedService<
       stream: args.server.stream,
       noResponderRetry: args.server.noResponderRetry,
       api: runtimeApi,
+      contractId: args.contractId,
+      contractDigest: args.contractDigest,
       eventConsumers: {
         metadata: args.contractEventConsumers,
         bindings: args.bindings.eventConsumers,
@@ -2009,6 +2040,14 @@ function toUnexpectedError(cause: unknown): UnexpectedError {
     : new UnexpectedError({ cause });
 }
 
+function serializeJobHandlerError(error: BaseError): string {
+  try {
+    return JSON.stringify(error.toSerializable());
+  } catch {
+    return error.message;
+  }
+}
+
 function okVoid(): Result<void, never> {
   return Result.ok(undefined);
 }
@@ -2403,6 +2442,8 @@ function createJobsFacade<
   TKv extends ContractKvMetadata = ContractKvMetadata,
 >(args: {
   serviceName: string;
+  contractId?: string;
+  contractDigest?: string;
   nc: NatsConnection;
   contractJobs: TJobs;
   client: Trellis<TTrellisApi, TKv, TJobs>;
@@ -2566,9 +2607,35 @@ function createJobsFacade<
                   },
                 );
 
-                const handled = (await handler(publicJob)).take();
+                const jobErrorContext = {
+                  jobType: queueType,
+                  requestId: job.context().requestId,
+                  service: args.serviceName,
+                  contractId: args.contractId,
+                  contractDigest: args.contractDigest,
+                  traceId: job.context().traceId,
+                };
+
+                let handled: unknown | Result<never, BaseError>;
+                try {
+                  handled = (await handler(publicJob)).take();
+                } catch (cause) {
+                  const annotatedError = annotateHandlerBoundaryError(
+                    cause,
+                    jobErrorContext,
+                  );
+                  throw InternalJobProcessError.failed(
+                    serializeJobHandlerError(annotatedError),
+                  );
+                }
                 if (isErr(handled)) {
-                  throw InternalJobProcessError.failed(handled.error.message);
+                  const annotatedError = annotateHandlerBoundaryError(
+                    handled.error,
+                    jobErrorContext,
+                  );
+                  throw InternalJobProcessError.failed(
+                    serializeJobHandlerError(annotatedError),
+                  );
                 }
                 return handled;
               },
@@ -2756,6 +2823,8 @@ export class TrellisService<
     this.#operationTransfer = operationTransfer;
     const jobs = createJobsFacade<TJobs, TTrellisApi, TKv>({
       serviceName: name,
+      contractId: health.contractId,
+      contractDigest: health.contractDigest,
       nc,
       contractJobs,
       client: handlerTrellis,
@@ -3097,6 +3166,9 @@ export class TrellisService<
           ...(await loadDefaultServiceRuntimeDeps()),
           ...deps,
         } satisfies TrellisServiceRuntimeDeps;
+        if (automaticTelemetryEnabled(args.telemetry)) {
+          runtimeDeps.initTelemetry?.(args.name);
+        }
         const auth = await createAuth({ sessionKeySeed: args.sessionKeySeed });
         const bootstrapLog = resolveServiceLogger(args.server?.log);
         const bootstrap = await fetchServiceBootstrapInfo({

package/src/server.ts

@@ -25,8 +25,13 @@ import {
 import { RemoteError } from "./errors/RemoteError.js";
 import type { LoggerLike } from "./globals.js";
 import { serverLogger } from "./server_logger.js";
+import {
+  recordTrellisError,
+  type TrellisErrorMetricAttributes,
+} from "./telemetry/mod.js";
 import {
   type AcceptedOperation,
+  annotateHandlerBoundaryError,
   type AnyTrellisAPI,
   type AuthRequestsValidateResponse,
   base64urlDecode,
@@ -199,6 +204,38 @@ function asOptionalStringRecordPointerValue(
   return ok(Object.fromEntries(entries) as Record<string, string>);
 }
 
+function traceIdFromTraceparent(
+  traceparent: string | undefined,
+): string | undefined {
+  const [version, traceId, parentId, flags, extra] = traceparent?.split("-") ??
+    [];
+  if (
+    extra !== undefined ||
+    !/^[0-9a-f]{2}$/u.test(version ?? "") ||
+    version === "ff" ||
+    !/^[0-9a-f]{32}$/u.test(traceId ?? "") ||
+    traceId === "00000000000000000000000000000000" ||
+    !/^[0-9a-f]{16}$/u.test(parentId ?? "") ||
+    parentId === "0000000000000000" ||
+    !/^[0-9a-f]{2}$/u.test(flags ?? "")
+  ) {
+    return undefined;
+  }
+  return traceId;
+}
+
+function recordOperationServerError(
+  error: unknown,
+  attributes: TrellisErrorMetricAttributes,
+): void {
+  recordTrellisError(error, {
+    messagingSystem: "nats",
+    surface: "operation",
+    direction: "server",
+    ...attributes,
+  });
+}
+
 export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
   #version?: string;
   #log: LoggerLike;
@@ -243,7 +280,7 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
         }),
       fail: (operationId, error) =>
         this.#applyOperationUpdate(operationId, "failed", {
-          patch: { error: { type: error.name, message: error.message } },
+          patch: { error: error.toSerializable() },
           event: { type: "failed" },
         }),
       cancel: (operationId) =>
@@ -506,7 +543,7 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
         }),
       fail: (error: BaseError) =>
         this.#applyControlledOperationUpdate(runtime, ctx, "failed", {
-          patch: { error: { type: error.name, message: error.message } },
+          patch: { error: error.toSerializable() },
           event: { type: "failed" },
         }),
       cancel: () => {
@@ -916,6 +953,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
       const trellisError = error instanceof BaseError
         ? error
         : new UnexpectedError({ cause: error });
+      recordOperationServerError(trellisError, {
+        operation,
+        phase: "control",
+      });
       msg.respond(JSON.stringify({
         kind: "error",
         error: trellisError.toSerializable(),
@@ -1210,7 +1251,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
           runtime.waiters.clear();
         };
 
-        const makeOperation = (runtime: RuntimeOperationRecord) => {
+        const makeOperation = (
+          runtime: RuntimeOperationRecord,
+          context: { requestId?: string; traceId?: string },
+        ) => {
           const ensureActive = () => {
             if (runtime.terminal) {
               return err(
@@ -1291,11 +1335,19 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
               AsyncResult.from((async () => {
                 const active = ensureActive();
                 if (active) return active;
+                const annotatedError = annotateHandlerBoundaryError(error, {
+                  operation: String(operation),
+                  requestId: context.requestId,
+                  service: this.name,
+                  contractId: this.contractId,
+                  contractDigest: this.contractDigest,
+                  traceId: context.traceId,
+                });
                 const snapshot = buildRuntimeOperationSnapshot(
                   runtime,
                   "failed",
                   {
-                    error: { type: error.name, message: error.message },
+                    error: annotatedError.toSerializable(),
                     completedAt: now(),
                   },
                 );
@@ -1507,6 +1559,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
             const validated = await authenticate(msg, true);
             const value = validated.take();
             if (isErr(value)) {
+              recordOperationServerError(value.error, {
+                operation: String(operation),
+                phase: "start",
+              });
               this.respondWithError(msg, value.error);
               continue;
             }
@@ -1514,15 +1570,20 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
             let transferSession: RuntimeOperationTransferSession | undefined;
             if (ctx.transfer) {
               if (!this.#transferSupport) {
+                const error = new UnexpectedError({
+                  cause: new Error(
+                    `Operation '${
+                      String(operation)
+                    }' declared transfer support but no runtime transfer support is configured`,
+                  ),
+                });
+                recordOperationServerError(error, {
+                  operation: String(operation),
+                  phase: "start",
+                });
                 this.respondWithError(
                   msg,
-                  new UnexpectedError({
-                    cause: new Error(
-                      `Operation '${
-                        String(operation)
-                      }' declared transfer support but no runtime transfer support is configured`,
-                    ),
-                  }),
+                  error,
                 );
                 continue;
               }
@@ -1534,6 +1595,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                 "key",
               ).take();
               if (isErr(key)) {
+                recordOperationServerError(key.error, {
+                  operation: String(operation),
+                  phase: "start",
+                });
                 this.respondWithError(msg, key.error);
                 continue;
               }
@@ -1543,6 +1608,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                 ctx.transfer.contentType,
               ).take();
               if (isErr(contentType)) {
+                recordOperationServerError(contentType.error, {
+                  operation: String(operation),
+                  phase: "start",
+                });
                 this.respondWithError(msg, contentType.error);
                 continue;
               }
@@ -1552,6 +1621,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                 ctx.transfer.metadata,
               ).take();
               if (isErr(metadata)) {
+                recordOperationServerError(metadata.error, {
+                  operation: String(operation),
+                  phase: "start",
+                });
                 this.respondWithError(msg, metadata.error);
                 continue;
               }
@@ -1569,6 +1642,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                   ...(metadata !== undefined ? { metadata } : {}),
                 }).take();
               if (isErr(openedTransferValue)) {
+                recordOperationServerError(openedTransferValue.error, {
+                  operation: String(operation),
+                  phase: "start",
+                });
                 this.respondWithError(msg, openedTransferValue.error);
                 continue;
               }
@@ -1636,7 +1713,13 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
             msg.respond(JSON.stringify(accepted));
 
             void (async () => {
-              const op = makeOperation(runtime);
+              const operationContext = {
+                requestId: msg.headers?.get("request-id"),
+                traceId: traceIdFromTraceparent(
+                  msg.headers?.get("traceparent"),
+                ),
+              };
+              const op = makeOperation(runtime, operationContext);
               try {
                 const handlerResult: unknown = await handler(
                   transferSession
@@ -1656,7 +1739,22 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                   ? handlerResult.take()
                   : handlerResult;
                 if (isErr(handlerOutcome)) {
-                  await op.fail(handlerOutcome.error);
+                  const error = annotateHandlerBoundaryError(
+                    handlerOutcome.error,
+                    {
+                      operation: String(operation),
+                      requestId: operationContext.requestId,
+                      service: this.name,
+                      contractId: this.contractId,
+                      contractDigest: this.contractDigest,
+                      traceId: operationContext.traceId,
+                    },
+                  );
+                  recordOperationServerError(error, {
+                    operation: String(operation),
+                    phase: "handler_result",
+                  });
+                  await op.fail(error);
                   return;
                 }
 
@@ -1676,7 +1774,19 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                   await op.complete(handlerOutcome);
                 }
               } catch (cause) {
-                await op.fail(new UnexpectedError({ cause }));
+                const error = annotateHandlerBoundaryError(cause, {
+                  operation: String(operation),
+                  requestId: operationContext.requestId,
+                  service: this.name,
+                  contractId: this.contractId,
+                  contractDigest: this.contractDigest,
+                  traceId: operationContext.traceId,
+                });
+                recordOperationServerError(error, {
+                  operation: String(operation),
+                  phase: "handler_throw",
+                });
+                await op.fail(error);
               }
             })();
           }

package/src/service/deno.ts

@@ -17,4 +17,5 @@ export {
   type Trellis,
   TrellisService,
   type TrellisServiceConnectOpts,
+  type TrellisServiceConnectTelemetryOpts,
 } from "../server/service.js";

package/src/service/mod.ts

@@ -54,6 +54,7 @@ export {
   type Trellis,
   TrellisService,
   type TrellisServiceConnectOpts,
+  type TrellisServiceConnectTelemetryOpts,
 } from "../server/service.js";
 export {
   createPostgresOutboxSchema,