@qlever-llc/trellis 0.10.11 → 0.10.12

package/src/errors/TrellisError.ts

@@ -1,20 +1,20 @@
 /**
  * Base class for all Trellis-specific errors.
- * Extends BaseError and relies on the traceId getter being configured via initTracing.
+ * Extends BaseError and relies on the traceId getter being configured via initTelemetry.
  */
 import { BaseError, type BaseErrorSchema } from "@qlever-llc/result";
 
 /**
  * Abstract base class for Trellis errors.
- * Trellis errors automatically include traceId when initTracing() has been called
+ * Trellis errors automatically include traceId when initTelemetry() has been called
  * and a span is active in the current context.
  *
- * The traceId integration is configured by the tracing module's initTracing() function,
+ * The traceId integration is configured by the telemetry module's initTelemetry() function,
  * which sets up BaseError.traceIdGetter to retrieve the traceId from the active span.
  */
 export abstract class TrellisError<
   TData extends BaseErrorSchema = BaseErrorSchema,
 > extends BaseError<TData> {
   // TrellisError inherits getTraceId() from BaseError which uses the static traceIdGetter.
-  // The traceIdGetter is configured by initTracing() in the tracing module.
+  // The traceIdGetter is configured by initTelemetry() in the telemetry module.
 }

package/src/server/internal_jobs/job-manager.ts

@@ -4,6 +4,7 @@ import {
   createMapCarrier,
   injectTraceContext,
 } from "../../telemetry/carrier.js";
+import { recordTrellisError } from "../../telemetry/mod.js";
 
 import {
   ActiveJob,
@@ -103,11 +104,22 @@ export class JobManager<TPayload = unknown, TResult = unknown> {
   ): Promise<void> {
     const binding = this.#getQueueBinding(type);
     const headers = headersFromJobContext(event.context);
-    await this.#context.nc.publish(
-      `${binding.publishPrefix}.${jobId}.${event.eventType}`,
-      new TextEncoder().encode(JSON.stringify(event)),
-      { headers },
-    );
+    try {
+      await this.#context.nc.publish(
+        `${binding.publishPrefix}.${jobId}.${event.eventType}`,
+        new TextEncoder().encode(JSON.stringify(event)),
+        { headers },
+      );
+    } catch (error) {
+      recordTrellisError(error, {
+        surface: "job",
+        direction: "worker",
+        operation: type,
+        phase: "publish",
+        messagingSystem: "nats",
+      });
+      throw error;
+    }
   }
 
   async create(
@@ -247,6 +259,12 @@ export class JobManager<TPayload = unknown, TResult = unknown> {
       if (error instanceof JobProcessError) {
         const detail = error.message;
         if (error.kind === "retryable") {
+          recordTrellisError(error, {
+            surface: "job",
+            direction: "worker",
+            operation: job.type,
+            phase: "handler_result",
+          });
           await this.#publishJobEvent(job.type, job.id, {
             jobId: job.id,
             service: job.service,
@@ -262,6 +280,12 @@ export class JobManager<TPayload = unknown, TResult = unknown> {
           return { outcome: "retry", tries, error: detail };
         }
 
+        recordTrellisError(error, {
+          surface: "job",
+          direction: "worker",
+          operation: job.type,
+          phase: "handler_result",
+        });
         await this.#publishJobEvent(job.type, job.id, {
           jobId: job.id,
           service: job.service,
@@ -277,6 +301,12 @@ export class JobManager<TPayload = unknown, TResult = unknown> {
         return { outcome: "failed", tries, error: detail };
       }
 
+      recordTrellisError(error, {
+        surface: "job",
+        direction: "worker",
+        operation: job.type,
+        phase: "runtime",
+      });
       throw error;
     }
   }

package/src/server/runtime.ts

@@ -20,8 +20,12 @@ export type NatsCredsAuthenticatorFn = (creds: Uint8Array) => unknown;
 
 export type ReadFileSyncFn = (path: string) => Uint8Array;
 
+/** Initializes telemetry for a service runtime. */
+export type InitTelemetryFn = (serviceName: string) => void;
+
 export type TrellisServiceRuntimeDeps = {
   connect: NatsConnectFn;
   credsAuthenticator?: NatsCredsAuthenticatorFn;
   readFileSync?: ReadFileSyncFn;
+  initTelemetry?: InitTelemetryFn;
 };

package/src/server/service.ts

@@ -128,6 +128,7 @@ import {
   observeNatsTrellisConnection,
   type TrellisConnection,
 } from "../connection.js";
+import { initTelemetry } from "../telemetry/init.js";
 
 type ExtraNatsConnectOpts = Omit<
   NatsConnectOpts,
@@ -431,6 +432,7 @@ async function loadDefaultServiceRuntimeDeps(): Promise<
 > {
   const transport = await loadDefaultRuntimeTransport();
   return {
+    initTelemetry,
     connect: (
       { servers, token, authenticator, inboxPrefix, ...extraOptions },
     ) =>
@@ -444,6 +446,12 @@ async function loadDefaultServiceRuntimeDeps(): Promise<
   };
 }
 
+function automaticTelemetryEnabled(
+  telemetry: TrellisServiceConnectTelemetryOpts | undefined,
+): boolean {
+  return telemetry !== false && telemetry?.enabled !== false;
+}
+
 const ServiceBootstrapReadySchema = Type.Object({
   status: Type.Literal("ready"),
   serverNow: Type.Integer(),
@@ -871,9 +879,20 @@ export type TrellisServiceConnectOpts<
   contract: ServiceContract<TOwnedApi, TTrellisApi>;
   name: string;
   sessionKeySeed: string;
+  /**
+   * Controls automatic telemetry initialization for this service connection.
+   * Enabled by default; pass `false` or `{ enabled: false }` to disable it.
+   */
+  telemetry?: TrellisServiceConnectTelemetryOpts;
   server?: TrellisServiceServerOpts;
 };
 
+/** Controls automatic telemetry initialization for `TrellisService.connect()`. */
+export type TrellisServiceConnectTelemetryOpts = false | {
+  /** Whether automatic telemetry initialization is enabled. Defaults to `true`. */
+  enabled?: boolean;
+};
+
 type ServiceKvFacade<TKv extends ContractKvMetadata> = {
   [K in keyof TKv]: TKv[K]["required"] extends false
     ? TypedKV<TKv[K]["schema"]> | undefined
@@ -1760,6 +1779,11 @@ export type TrellisServiceConnectArgs<
   contract: TContract;
   name: string;
   sessionKeySeed: string;
+  /**
+   * Controls automatic telemetry initialization for this service connection.
+   * Enabled by default; pass `false` or `{ enabled: false }` to disable it.
+   */
+  telemetry?: TrellisServiceConnectTelemetryOpts;
   server?: TrellisServiceServerOpts;
 };
 
@@ -3142,6 +3166,9 @@ export class TrellisService<
           ...(await loadDefaultServiceRuntimeDeps()),
           ...deps,
         } satisfies TrellisServiceRuntimeDeps;
+        if (automaticTelemetryEnabled(args.telemetry)) {
+          runtimeDeps.initTelemetry?.(args.name);
+        }
         const auth = await createAuth({ sessionKeySeed: args.sessionKeySeed });
         const bootstrapLog = resolveServiceLogger(args.server?.log);
         const bootstrap = await fetchServiceBootstrapInfo({

package/src/server.ts

@@ -25,6 +25,10 @@ import {
 import { RemoteError } from "./errors/RemoteError.js";
 import type { LoggerLike } from "./globals.js";
 import { serverLogger } from "./server_logger.js";
+import {
+  recordTrellisError,
+  type TrellisErrorMetricAttributes,
+} from "./telemetry/mod.js";
 import {
   type AcceptedOperation,
   annotateHandlerBoundaryError,
@@ -220,6 +224,18 @@ function traceIdFromTraceparent(
   return traceId;
 }
 
+function recordOperationServerError(
+  error: unknown,
+  attributes: TrellisErrorMetricAttributes,
+): void {
+  recordTrellisError(error, {
+    messagingSystem: "nats",
+    surface: "operation",
+    direction: "server",
+    ...attributes,
+  });
+}
+
 export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
   #version?: string;
   #log: LoggerLike;
@@ -937,6 +953,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
       const trellisError = error instanceof BaseError
         ? error
         : new UnexpectedError({ cause: error });
+      recordOperationServerError(trellisError, {
+        operation,
+        phase: "control",
+      });
       msg.respond(JSON.stringify({
         kind: "error",
         error: trellisError.toSerializable(),
@@ -1539,6 +1559,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
             const validated = await authenticate(msg, true);
             const value = validated.take();
             if (isErr(value)) {
+              recordOperationServerError(value.error, {
+                operation: String(operation),
+                phase: "start",
+              });
               this.respondWithError(msg, value.error);
               continue;
             }
@@ -1546,15 +1570,20 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
             let transferSession: RuntimeOperationTransferSession | undefined;
             if (ctx.transfer) {
               if (!this.#transferSupport) {
+                const error = new UnexpectedError({
+                  cause: new Error(
+                    `Operation '${
+                      String(operation)
+                    }' declared transfer support but no runtime transfer support is configured`,
+                  ),
+                });
+                recordOperationServerError(error, {
+                  operation: String(operation),
+                  phase: "start",
+                });
                 this.respondWithError(
                   msg,
-                  new UnexpectedError({
-                    cause: new Error(
-                      `Operation '${
-                        String(operation)
-                      }' declared transfer support but no runtime transfer support is configured`,
-                    ),
-                  }),
+                  error,
                 );
                 continue;
               }
@@ -1566,6 +1595,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                 "key",
               ).take();
               if (isErr(key)) {
+                recordOperationServerError(key.error, {
+                  operation: String(operation),
+                  phase: "start",
+                });
                 this.respondWithError(msg, key.error);
                 continue;
               }
@@ -1575,6 +1608,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                 ctx.transfer.contentType,
               ).take();
               if (isErr(contentType)) {
+                recordOperationServerError(contentType.error, {
+                  operation: String(operation),
+                  phase: "start",
+                });
                 this.respondWithError(msg, contentType.error);
                 continue;
               }
@@ -1584,6 +1621,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                 ctx.transfer.metadata,
               ).take();
               if (isErr(metadata)) {
+                recordOperationServerError(metadata.error, {
+                  operation: String(operation),
+                  phase: "start",
+                });
                 this.respondWithError(msg, metadata.error);
                 continue;
               }
@@ -1601,6 +1642,10 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                   ...(metadata !== undefined ? { metadata } : {}),
                 }).take();
               if (isErr(openedTransferValue)) {
+                recordOperationServerError(openedTransferValue.error, {
+                  operation: String(operation),
+                  phase: "start",
+                });
                 this.respondWithError(msg, openedTransferValue.error);
                 continue;
               }
@@ -1694,7 +1739,7 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                   ? handlerResult.take()
                   : handlerResult;
                 if (isErr(handlerOutcome)) {
-                  await op.fail(annotateHandlerBoundaryError(
+                  const error = annotateHandlerBoundaryError(
                     handlerOutcome.error,
                     {
                       operation: String(operation),
@@ -1704,7 +1749,12 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                       contractDigest: this.contractDigest,
                       traceId: operationContext.traceId,
                     },
-                  ));
+                  );
+                  recordOperationServerError(error, {
+                    operation: String(operation),
+                    phase: "handler_result",
+                  });
+                  await op.fail(error);
                   return;
                 }
 
@@ -1724,14 +1774,19 @@ export class TrellisServiceRuntime extends Trellis<TrellisAPI, TrellisMode> {
                   await op.complete(handlerOutcome);
                 }
               } catch (cause) {
-                await op.fail(annotateHandlerBoundaryError(cause, {
+                const error = annotateHandlerBoundaryError(cause, {
                   operation: String(operation),
                   requestId: operationContext.requestId,
                   service: this.name,
                   contractId: this.contractId,
                   contractDigest: this.contractDigest,
                   traceId: operationContext.traceId,
-                }));
+                });
+                recordOperationServerError(error, {
+                  operation: String(operation),
+                  phase: "handler_throw",
+                });
+                await op.fail(error);
               }
             })();
           }

package/src/service/deno.ts

@@ -17,4 +17,5 @@ export {
   type Trellis,
   TrellisService,
   type TrellisServiceConnectOpts,
+  type TrellisServiceConnectTelemetryOpts,
 } from "../server/service.js";

package/src/service/mod.ts

@@ -54,6 +54,7 @@ export {
   type Trellis,
   TrellisService,
   type TrellisServiceConnectOpts,
+  type TrellisServiceConnectTelemetryOpts,
 } from "../server/service.js";
 export {
   createPostgresOutboxSchema,

package/src/service/node.ts

@@ -17,4 +17,5 @@ export {
   type Trellis,
   TrellisService,
   type TrellisServiceConnectOpts,
+  type TrellisServiceConnectTelemetryOpts,
 } from "../server/service.js";

package/src/service/outbox_inbox.ts

@@ -3,6 +3,7 @@ import { type AsyncResult, type BaseError, isErr } from "@qlever-llc/result";
 import { type StaticDecode, Type } from "typebox";
 import type { PreparedTrellisEvent, Trellis } from "../trellis.js";
 import { TypedKV } from "../kv.js";
+import { recordTrellisError } from "../telemetry/mod.js";
 
 export type OutboxMessageState = "pending" | "dispatched" | "failed";
 
@@ -646,6 +647,12 @@ export class OutboxDispatcher {
         retryDelayMs: this.#retryDelayMs(),
       });
     } catch (error) {
+      recordTrellisError(error, {
+        surface: "outbox",
+        direction: "dispatcher",
+        operation: "batch",
+        phase: "dispatch",
+      });
       this.#scheduleRetryWakeup();
       try {
         this.#options.onError?.(error);
@@ -693,6 +700,13 @@ export async function dispatchOutbox(
     );
     const value = result.take();
     if (isErr(value)) {
+      recordTrellisError(value.error, {
+        surface: "outbox",
+        direction: "dispatcher",
+        operation: message.event,
+        phase: "publish",
+        messagingSystem: "nats",
+      });
       failed += 1;
       await repository.markFailed(message.id, {
         error: value.error.message,

package/src/telemetry/core.ts

@@ -8,7 +8,7 @@ import {
   type Tracer,
 } from "@opentelemetry/api";
 
-export function getTracer(scope = "@qlever-llc/trellis/tracing"): Tracer {
+export function getTracer(scope = "@qlever-llc/trellis/telemetry"): Tracer {
   return trace.getTracer(scope);
 }
 

package/src/telemetry/env.ts

@@ -13,7 +13,11 @@ type ProcessLike = {
 export function getEnv(key: string): string | undefined {
   const deno = dntShim.dntGlobalThis as typeof dntShim.dntGlobalThis & { Deno?: DenoLike };
   if (deno.Deno?.env?.get) {
-    return deno.Deno.env.get(key);
+    try {
+      return deno.Deno.env.get(key);
+    } catch {
+      return undefined;
+    }
   }
 
   const processGlobal = dntShim.dntGlobalThis as typeof dntShim.dntGlobalThis & {

package/src/telemetry/init.ts

@@ -0,0 +1,8 @@
+import { configureErrorTraceId } from "./result.js";
+import { initTelemetryRuntime } from "./runtime.js";
+
+/** Initializes Trellis telemetry for a service runtime. */
+export function initTelemetry(serviceName: string): void {
+  configureErrorTraceId();
+  initTelemetryRuntime(serviceName);
+}