@temporalio/interceptors-opentelemetry 1.13.1 → 1.14.0

package/src/client/index.ts

@@ -1,6 +1,31 @@
 import * as otel from '@opentelemetry/api';
-import { Next, WorkflowClientInterceptor, WorkflowSignalInput, WorkflowStartInput } from '@temporalio/client';
-import { instrument, headersWithContext, RUN_ID_ATTR_KEY } from '../instrumentation';
+import type {
+  Next,
+  WorkflowSignalInput,
+  WorkflowSignalWithStartInput,
+  WorkflowStartInput,
+  WorkflowStartOutput,
+  WorkflowStartUpdateInput,
+  WorkflowStartUpdateOutput,
+  WorkflowStartUpdateWithStartInput,
+  WorkflowStartUpdateWithStartOutput,
+  WorkflowQueryInput,
+  WorkflowTerminateInput,
+  WorkflowCancelInput,
+  WorkflowDescribeInput,
+  WorkflowClientInterceptor,
+  TerminateWorkflowExecutionResponse,
+  RequestCancelWorkflowExecutionResponse,
+  DescribeWorkflowExecutionResponse,
+} from '@temporalio/client';
+import {
+  instrument,
+  headersWithContext,
+  RUN_ID_ATTR_KEY,
+  WORKFLOW_ID_ATTR_KEY,
+  UPDATE_ID_ATTR_KEY,
+  TERMINATE_REASON_ATTR_KEY,
+} from '../instrumentation';
 import { SpanName, SPAN_DELIMITER } from '../workflow';
 
 export interface InterceptorOptions {
@@ -24,7 +49,8 @@ export class OpenTelemetryWorkflowClientInterceptor implements WorkflowClientInt
       tracer: this.tracer,
       spanName: `${SpanName.WORKFLOW_START}${SPAN_DELIMITER}${input.workflowType}`,
       fn: async (span) => {
-        const headers = await headersWithContext(input.headers);
+        const headers = headersWithContext(input.headers);
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.options.workflowId);
         const runId = await next({ ...input, headers });
         span.setAttribute(RUN_ID_ATTR_KEY, runId);
         return runId;
@@ -36,10 +62,157 @@ export class OpenTelemetryWorkflowClientInterceptor implements WorkflowClientInt
     return await instrument({
       tracer: this.tracer,
       spanName: `${SpanName.WORKFLOW_SIGNAL}${SPAN_DELIMITER}${input.signalName}`,
-      fn: async () => {
-        const headers = await headersWithContext(input.headers);
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId);
+        const headers = headersWithContext(input.headers);
         await next({ ...input, headers });
       },
     });
   }
+
+  async startWithDetails(
+    input: WorkflowStartInput,
+    next: Next<WorkflowClientInterceptor, 'startWithDetails'>
+  ): Promise<WorkflowStartOutput> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: `${SpanName.WORKFLOW_START}${SPAN_DELIMITER}${input.workflowType}`,
+      fn: async (span) => {
+        const headers = headersWithContext(input.headers);
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.options.workflowId);
+        const output = await next({ ...input, headers });
+        span.setAttribute(RUN_ID_ATTR_KEY, output.runId);
+        return output;
+      },
+    });
+  }
+
+  async startUpdate(
+    input: WorkflowStartUpdateInput,
+    next: Next<WorkflowClientInterceptor, 'startUpdate'>
+  ): Promise<WorkflowStartUpdateOutput> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: `${SpanName.WORKFLOW_START_UPDATE}${SPAN_DELIMITER}${input.updateName}`,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId);
+        if (input.options.updateId) {
+          span.setAttribute(UPDATE_ID_ATTR_KEY, input.options.updateId);
+        }
+        const headers = headersWithContext(input.headers);
+        const output = await next({ ...input, headers });
+        span.setAttribute(RUN_ID_ATTR_KEY, output.workflowRunId);
+        return output;
+      },
+    });
+  }
+
+  async startUpdateWithStart(
+    input: WorkflowStartUpdateWithStartInput,
+    next: Next<WorkflowClientInterceptor, 'startUpdateWithStart'>
+  ): Promise<WorkflowStartUpdateWithStartOutput> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: `${SpanName.WORKFLOW_UPDATE_WITH_START}${SPAN_DELIMITER}${input.updateName}`,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowStartOptions.workflowId);
+        if (input.updateOptions.updateId) {
+          span.setAttribute(UPDATE_ID_ATTR_KEY, input.updateOptions.updateId);
+        }
+        const workflowStartHeaders = headersWithContext(input.workflowStartHeaders);
+        const updateHeaders = headersWithContext(input.updateHeaders);
+        const output = await next({ ...input, workflowStartHeaders, updateHeaders });
+        if (output.workflowExecution.runId) {
+          span.setAttribute(RUN_ID_ATTR_KEY, output.workflowExecution.runId);
+        }
+        return output;
+      },
+    });
+  }
+
+  async signalWithStart(
+    input: WorkflowSignalWithStartInput,
+    next: Next<WorkflowClientInterceptor, 'signalWithStart'>
+  ): Promise<string> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: `${SpanName.WORKFLOW_SIGNAL_WITH_START}${SPAN_DELIMITER}${input.workflowType}`,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.options.workflowId);
+        const headers = headersWithContext(input.headers);
+        const runId = await next({ ...input, headers });
+        span.setAttribute(RUN_ID_ATTR_KEY, runId);
+        return runId;
+      },
+    });
+  }
+
+  async query(input: WorkflowQueryInput, next: Next<WorkflowClientInterceptor, 'query'>): Promise<unknown> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: `${SpanName.WORKFLOW_QUERY}${SPAN_DELIMITER}${input.queryType}`,
+      fn: async (span) => {
+        const headers = headersWithContext(input.headers);
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId);
+        if (input.workflowExecution.runId) {
+          span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId);
+        }
+        return await next({ ...input, headers });
+      },
+    });
+  }
+
+  async terminate(
+    input: WorkflowTerminateInput,
+    next: Next<WorkflowClientInterceptor, 'terminate'>
+  ): Promise<TerminateWorkflowExecutionResponse> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: SpanName.WORKFLOW_TERMINATE,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId);
+        if (input.workflowExecution.runId) {
+          span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId);
+        }
+        if (input.reason) {
+          span.setAttribute(TERMINATE_REASON_ATTR_KEY, input.reason);
+        }
+        return await next(input);
+      },
+    });
+  }
+
+  async cancel(
+    input: WorkflowCancelInput,
+    next: Next<WorkflowClientInterceptor, 'cancel'>
+  ): Promise<RequestCancelWorkflowExecutionResponse> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: SpanName.WORKFLOW_CANCEL,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId);
+        if (input.workflowExecution.runId) {
+          span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId);
+        }
+        return await next(input);
+      },
+    });
+  }
+
+  async describe(
+    input: WorkflowDescribeInput,
+    next: Next<WorkflowClientInterceptor, 'describe'>
+  ): Promise<DescribeWorkflowExecutionResponse> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: SpanName.WORKFLOW_DESCRIBE,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId);
+        if (input.workflowExecution.runId) {
+          span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId);
+        }
+        return await next(input);
+      },
+    });
+  }
 }

package/src/instrumentation.ts

@@ -3,19 +3,38 @@
  * @module
  */
 import * as otel from '@opentelemetry/api';
-import { ApplicationFailure, ApplicationFailureCategory, Headers, defaultPayloadConverter } from '@temporalio/common';
+import {
+  type Headers,
+  ApplicationFailure,
+  ApplicationFailureCategory,
+  defaultPayloadConverter,
+} from '@temporalio/common';
 
 /** Default trace header for opentelemetry interceptors */
 export const TRACE_HEADER = '_tracer-data';
 /** As in workflow run id */
 export const RUN_ID_ATTR_KEY = 'run_id';
+/** As in workflow id */
+export const WORKFLOW_ID_ATTR_KEY = 'temporalWorkflowId';
+/** As in activity id */
+export const ACTIVITY_ID_ATTR_KEY = 'temporalActivityId';
+/** As in update id */
+export const UPDATE_ID_ATTR_KEY = 'temporalUpdateId';
+/** As in termination reason */
+export const TERMINATE_REASON_ATTR_KEY = 'temporalTerminateReason';
+/** As in Nexus service */
+export const NEXUS_SERVICE_ATTR_KEY = 'temporalNexusService';
+/** As in Nexus operation */
+export const NEXUS_OPERATION_ATTR_KEY = 'temporalNexusOperation';
+/** As in Nexus endpoint */
+export const NEXUS_ENDPOINT_ATTR_KEY = 'temporalNexusEndpoint';
 
 const payloadConverter = defaultPayloadConverter;
 
 /**
  * If found, return an otel Context deserialized from the provided headers
  */
-export async function extractContextFromHeaders(headers: Headers): Promise<otel.Context | undefined> {
+export function extractContextFromHeaders(headers: Headers): otel.Context | undefined {
   const encodedSpanContext = headers[TRACE_HEADER];
   if (encodedSpanContext === undefined) {
     return undefined;
@@ -27,7 +46,7 @@ export async function extractContextFromHeaders(headers: Headers): Promise<otel.
 /**
  * Given headers, return new headers with the current otel context inserted
  */
-export async function headersWithContext(headers: Headers): Promise<Headers> {
+export function headersWithContext(headers: Headers): Headers {
   const carrier = {};
   otel.propagation.inject(otel.context.active(), carrier, otel.defaultTextMapSetter);
   return { ...headers, [TRACE_HEADER]: payloadConverter.toPayload(carrier) };
@@ -43,20 +62,41 @@ async function wrapWithSpan<T>(
     span.setStatus({ code: otel.SpanStatusCode.OK });
     return ret;
   } catch (err: any) {
-    const isBenignErr = err instanceof ApplicationFailure && err.category === ApplicationFailureCategory.BENIGN;
-    if (acceptableErrors === undefined || !acceptableErrors(err)) {
-      const statusCode = isBenignErr ? otel.SpanStatusCode.UNSET : otel.SpanStatusCode.ERROR;
-      span.setStatus({ code: statusCode, message: (err as Error).message ?? String(err) });
-      span.recordException(err);
-    } else {
-      span.setStatus({ code: otel.SpanStatusCode.OK });
-    }
+    maybeAddErrorToSpan(err, span, acceptableErrors);
     throw err;
   } finally {
     span.end();
   }
 }
 
+function wrapWithSpanSync<T>(
+  span: otel.Span,
+  fn: (span: otel.Span) => T,
+  acceptableErrors?: (err: unknown) => boolean
+): T {
+  try {
+    const ret = fn(span);
+    span.setStatus({ code: otel.SpanStatusCode.OK });
+    return ret;
+  } catch (err: any) {
+    maybeAddErrorToSpan(err, span, acceptableErrors);
+    throw err;
+  } finally {
+    span.end();
+  }
+}
+
+function maybeAddErrorToSpan(err: any, span: otel.Span, acceptableErrors?: (err: unknown) => boolean): void {
+  const isBenignErr = err instanceof ApplicationFailure && err.category === ApplicationFailureCategory.BENIGN;
+  if (acceptableErrors === undefined || !acceptableErrors(err)) {
+    const statusCode = isBenignErr ? otel.SpanStatusCode.UNSET : otel.SpanStatusCode.ERROR;
+    span.setStatus({ code: statusCode, message: (err as Error).message ?? String(err) });
+    span.recordException(err);
+  } else {
+    span.setStatus({ code: otel.SpanStatusCode.OK });
+  }
+}
+
 export interface InstrumentOptions<T> {
   tracer: otel.Tracer;
   spanName: string;
@@ -65,6 +105,8 @@ export interface InstrumentOptions<T> {
   acceptableErrors?: (err: unknown) => boolean;
 }
 
+export type InstrumentOptionsSync<T> = Omit<InstrumentOptions<T>, 'fn'> & { fn: (span: otel.Span) => T };
+
 /**
  * Wraps `fn` in a span which ends when function returns or throws
  */
@@ -82,3 +124,12 @@ export async function instrument<T>({
   }
   return await tracer.startActiveSpan(spanName, async (span) => await wrapWithSpan(span, fn, acceptableErrors));
 }
+
+export function instrumentSync<T>({ tracer, spanName, fn, context, acceptableErrors }: InstrumentOptionsSync<T>): T {
+  if (context) {
+    return otel.context.with(context, () => {
+      return tracer.startActiveSpan(spanName, (span) => wrapWithSpanSync(span, fn, acceptableErrors));
+    });
+  }
+  return tracer.startActiveSpan(spanName, (span) => wrapWithSpanSync(span, fn, acceptableErrors));
+}

package/src/worker/index.ts

@@ -1,17 +1,24 @@
 import * as otel from '@opentelemetry/api';
-import { Resource } from '@opentelemetry/resources';
-import { ReadableSpan, SpanExporter } from '@opentelemetry/sdk-trace-base';
-import { Context as ActivityContext } from '@temporalio/activity';
-import {
-  ActivityExecuteInput,
+import type { Resource } from '@opentelemetry/resources';
+import type { ReadableSpan, SpanExporter } from '@opentelemetry/sdk-trace-base';
+import type { Context as ActivityContext } from '@temporalio/activity';
+import type {
+  Next,
   ActivityInboundCallsInterceptor,
   ActivityOutboundCallsInterceptor,
-  GetLogAttributesInput,
   InjectedSink,
-  Next,
+  GetLogAttributesInput,
+  GetMetricTagsInput,
+  ActivityExecuteInput,
 } from '@temporalio/worker';
-import { instrument, extractContextFromHeaders } from '../instrumentation';
-import { OpenTelemetryWorkflowExporter, SerializableSpan, SpanName, SPAN_DELIMITER } from '../workflow';
+import {
+  instrument,
+  extractContextFromHeaders,
+  WORKFLOW_ID_ATTR_KEY,
+  RUN_ID_ATTR_KEY,
+  ACTIVITY_ID_ATTR_KEY,
+} from '../instrumentation';
+import { type OpenTelemetryWorkflowExporter, type SerializableSpan, SpanName, SPAN_DELIMITER } from '../workflow';
 
 export interface InterceptorOptions {
   readonly tracer?: otel.Tracer;
@@ -34,16 +41,26 @@ export class OpenTelemetryActivityInboundInterceptor implements ActivityInboundC
   }
 
   async execute(input: ActivityExecuteInput, next: Next<ActivityInboundCallsInterceptor, 'execute'>): Promise<unknown> {
-    const context = await extractContextFromHeaders(input.headers);
+    const context = extractContextFromHeaders(input.headers);
     const spanName = `${SpanName.ACTIVITY_EXECUTE}${SPAN_DELIMITER}${this.ctx.info.activityType}`;
-    return await instrument({ tracer: this.tracer, spanName, fn: () => next(input), context });
+    return await instrument({
+      tracer: this.tracer,
+      spanName,
+      fn: (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, this.ctx.info.workflowExecution.workflowId);
+        span.setAttribute(RUN_ID_ATTR_KEY, this.ctx.info.workflowExecution.runId);
+        span.setAttribute(ACTIVITY_ID_ATTR_KEY, this.ctx.info.activityId);
+        return next(input);
+      },
+      context,
+    });
   }
 }
 
 /**
- * Intercepts calls to emit logs from an Activity.
+ * Intercepts calls to emit logs and metrics from an Activity.
  *
- * Attach OpenTelemetry context tracing attributes to emitted log messages, if appropriate.
+ * Attach OpenTelemetry context tracing attributes to emitted log messages and metrics, if appropriate.
  */
 export class OpenTelemetryActivityOutboundInterceptor implements ActivityOutboundCallsInterceptor {
   constructor(protected readonly ctx: ActivityContext) {}
@@ -65,6 +82,24 @@ export class OpenTelemetryActivityOutboundInterceptor implements ActivityOutboun
       return next(input);
     }
   }
+
+  public getMetricTags(
+    input: GetMetricTagsInput,
+    next: Next<ActivityOutboundCallsInterceptor, 'getMetricTags'>
+  ): GetMetricTagsInput {
+    const span = otel.trace.getSpan(otel.context.active());
+    const spanContext = span?.spanContext();
+    if (spanContext && otel.isSpanContextValid(spanContext)) {
+      return next({
+        trace_id: spanContext.traceId,
+        span_id: spanContext.spanId,
+        trace_flags: `0${spanContext.traceFlags.toString(16)}`,
+        ...input,
+      });
+    } else {
+      return next(input);
+    }
+  }
 }
 
 /**

package/src/workflow/context-manager.ts

@@ -1,11 +1,19 @@
 import * as otel from '@opentelemetry/api';
-import { AsyncLocalStorage } from '@temporalio/workflow';
+import { ensureWorkflowModuleLoaded, getWorkflowModuleIfAvailable } from './workflow-module-loader';
+
+const AsyncLocalStorage = getWorkflowModuleIfAvailable()?.AsyncLocalStorage;
 
 export class ContextManager implements otel.ContextManager {
-  protected storage = new AsyncLocalStorage<otel.Context>();
+  // If `@temporalio/workflow` is not available, ignore for now.
+  // When ContextManager is constructed module resolution error will be thrown.
+  protected storage = AsyncLocalStorage ? new AsyncLocalStorage<otel.Context>() : undefined;
+
+  public constructor() {
+    ensureWorkflowModuleLoaded();
+  }
 
   active(): otel.Context {
-    return this.storage.getStore() || otel.ROOT_CONTEXT;
+    return this.storage!.getStore() || otel.ROOT_CONTEXT;
   }
 
   bind<T>(context: otel.Context, target: T): T {
@@ -36,7 +44,7 @@ export class ContextManager implements otel.ContextManager {
   }
 
   disable(): this {
-    this.storage.disable();
+    this.storage!.disable();
     return this;
   }
 
@@ -47,6 +55,6 @@ export class ContextManager implements otel.ContextManager {
     ...args: A
   ): ReturnType<F> {
     const cb = thisArg == null ? fn : fn.bind(thisArg);
-    return this.storage.run(context, cb, ...args);
+    return this.storage!.run(context, cb, ...args);
   }
 }

package/src/workflow/definitions.ts

@@ -54,6 +54,56 @@ export enum SpanName {
    */
   WORKFLOW_SIGNAL_WITH_START = 'SignalWithStartWorkflow',
 
+  /**
+   * Workflow is queried
+   */
+  WORKFLOW_QUERY = 'QueryWorkflow',
+
+  /**
+   * Workflow update is started by client
+   */
+  WORKFLOW_START_UPDATE = 'StartWorkflowUpdate',
+
+  /**
+   * Workflow is started with an update
+   */
+  WORKFLOW_UPDATE_WITH_START = 'UpdateWithStartWorkflow',
+
+  /**
+   * Workflow handles an incoming signal
+   */
+  WORKFLOW_HANDLE_SIGNAL = 'HandleSignal',
+
+  /**
+   * Workflow handles an incoming query
+   */
+  WORKFLOW_HANDLE_QUERY = 'HandleQuery',
+
+  /**
+   * Workflow handles an incoming update
+   */
+  WORKFLOW_HANDLE_UPDATE = 'HandleUpdate',
+
+  /**
+   * Workflow validates an incoming update
+   */
+  WORKFLOW_VALIDATE_UPDATE = 'ValidateUpdate',
+
+  /**
+   * Workflow is terminated
+   */
+  WORKFLOW_TERMINATE = 'TerminateWorkflow',
+
+  /**
+   * Workflow is cancelled
+   */
+  WORKFLOW_CANCEL = 'CancelWorkflow',
+
+  /**
+   * Workflow is described
+   */
+  WORKFLOW_DESCRIBE = 'DescribeWorkflow',
+
   /**
    * Workflow run is executing
    */
@@ -74,6 +124,10 @@ export enum SpanName {
    * Workflow is continuing as new
    */
   CONTINUE_AS_NEW = 'ContinueAsNew',
+  /**
+   * Nexus operation is started
+   */
+  NEXUS_OPERATION_START = 'StartNexusOperation',
 }
 
 export const SPAN_DELIMITER = ':';