@temporalio/interceptors-opentelemetry-v2 1.19.0

package/src/worker/index.ts

@@ -0,0 +1,264 @@
+import * as otel from '@opentelemetry/api';
+import { createTraceState } from '@opentelemetry/api';
+import type { Resource } from '@opentelemetry/resources';
+import type { ReadableSpan, SpanProcessor } from '@opentelemetry/sdk-trace-base';
+import type * as nexus from 'nexus-rpc';
+import type { Context as ActivityContext } from '@temporalio/activity';
+import { CompleteAsyncError } from '@temporalio/common';
+import type {
+  Next,
+  ActivityInboundCallsInterceptor,
+  ActivityOutboundCallsInterceptor,
+  NexusInboundCallsInterceptor,
+  NexusOutboundCallsInterceptor,
+  NexusStartOperationInput,
+  NexusStartOperationOutput,
+  NexusCancelOperationInput,
+  InjectedSink,
+  GetLogAttributesInput,
+  GetMetricTagsInput,
+  ActivityExecuteInput,
+} from '@temporalio/worker';
+import {
+  instrument,
+  extractContextFromHeaders,
+  extractContextFromNexusHeaders,
+  WORKFLOW_ID_ATTR_KEY,
+  RUN_ID_ATTR_KEY,
+  ACTIVITY_ID_ATTR_KEY,
+  NEXUS_SERVICE_ATTR_KEY,
+  NEXUS_OPERATION_ATTR_KEY,
+} from '../instrumentation';
+import {
+  type OpenTelemetryWorkflowExporter,
+  type SerializableSpan,
+  SpanName,
+  SPAN_DELIMITER,
+} from '../workflow/definitions';
+
+export interface InterceptorOptions {
+  readonly tracer?: otel.Tracer;
+}
+
+/**
+ * Intercepts calls to start an Activity.
+ *
+ * Wraps the operation in an opentelemetry Span and links it to a parent Span context if one is
+ * provided in the Activity input headers.
+ *
+ * @experimental This interceptor is experimental and APIs may change without notice.
+ */
+export class OpenTelemetryActivityInboundInterceptor implements ActivityInboundCallsInterceptor {
+  protected readonly tracer: otel.Tracer;
+
+  constructor(
+    protected readonly ctx: ActivityContext,
+    options?: InterceptorOptions
+  ) {
+    this.tracer = options?.tracer ?? otel.trace.getTracer('@temporalio/interceptor-activity');
+  }
+
+  async execute(input: ActivityExecuteInput, next: Next<ActivityInboundCallsInterceptor, 'execute'>): Promise<unknown> {
+    const context = extractContextFromHeaders(input.headers);
+    const spanName = `${SpanName.ACTIVITY_EXECUTE}${SPAN_DELIMITER}${this.ctx.info.activityType}`;
+    return await instrument({
+      tracer: this.tracer,
+      spanName,
+      fn: (span) => {
+        span.setAttribute(ACTIVITY_ID_ATTR_KEY, this.ctx.info.activityId);
+        if (this.ctx.info.inWorkflow) {
+          span.setAttribute(WORKFLOW_ID_ATTR_KEY, this.ctx.info.workflowExecution!.workflowId);
+          span.setAttribute(RUN_ID_ATTR_KEY, this.ctx.info.workflowExecution!.runId);
+        }
+        return next(input);
+      },
+      context,
+      acceptableErrors: (err) => err instanceof CompleteAsyncError,
+    });
+  }
+}
+
+/**
+ * Intercepts calls to emit logs and metrics from an Activity.
+ *
+ * Attach OpenTelemetry context tracing attributes to emitted log messages, if appropriate.
+ *
+ * @experimental This interceptor is experimental and APIs may change without notice.
+ */
+export class OpenTelemetryActivityOutboundInterceptor implements ActivityOutboundCallsInterceptor {
+  constructor(protected readonly ctx: ActivityContext) {}
+
+  public getLogAttributes(
+    input: GetLogAttributesInput,
+    next: Next<ActivityOutboundCallsInterceptor, 'getLogAttributes'>
+  ): Record<string, unknown> {
+    const span = otel.trace.getSpan(otel.context.active());
+    const spanContext = span?.spanContext();
+    if (spanContext && otel.isSpanContextValid(spanContext)) {
+      return next({
+        trace_id: spanContext.traceId,
+        span_id: spanContext.spanId,
+        trace_flags: `0${spanContext.traceFlags.toString(16)}`,
+        ...input,
+      });
+    } else {
+      return next(input);
+    }
+  }
+
+  public getMetricTags(
+    input: GetMetricTagsInput,
+    next: Next<ActivityOutboundCallsInterceptor, 'getMetricTags'>
+  ): GetMetricTagsInput {
+    return next(input);
+  }
+}
+
+/**
+ * Intercepts inbound Nexus Operation calls on the worker.
+ *
+ * Wraps `startOperation` and `cancelOperation` in opentelemetry Spans, linking to a parent Span context
+ * if one is provided in the Nexus request headers.
+ *
+ * @experimental This interceptor is experimental and APIs may change without notice.
+ */
+export class OpenTelemetryNexusInboundInterceptor implements NexusInboundCallsInterceptor {
+  protected readonly tracer: otel.Tracer;
+
+  constructor(
+    protected readonly ctx: nexus.OperationContext,
+    options?: InterceptorOptions
+  ) {
+    this.tracer = options?.tracer ?? otel.trace.getTracer('@temporalio/interceptor-nexus');
+  }
+
+  async startOperation(
+    input: NexusStartOperationInput,
+    next: Next<NexusInboundCallsInterceptor, 'startOperation'>
+  ): Promise<NexusStartOperationOutput> {
+    const context = extractContextFromNexusHeaders(input.ctx.headers);
+    const spanName = `${SpanName.NEXUS_START_OPERATION_EXECUTE}${SPAN_DELIMITER}${input.ctx.service}/${input.ctx.operation}`;
+    return await instrument({
+      tracer: this.tracer,
+      spanName,
+      fn: (span) => {
+        span.setAttribute(NEXUS_SERVICE_ATTR_KEY, input.ctx.service);
+        span.setAttribute(NEXUS_OPERATION_ATTR_KEY, input.ctx.operation);
+        return next(input);
+      },
+      context,
+    });
+  }
+
+  async cancelOperation(
+    input: NexusCancelOperationInput,
+    next: Next<NexusInboundCallsInterceptor, 'cancelOperation'>
+  ): Promise<void> {
+    const context = extractContextFromNexusHeaders(input.ctx.headers);
+    const spanName = `${SpanName.NEXUS_CANCEL_OPERATION_EXECUTE}${SPAN_DELIMITER}${input.ctx.service}/${input.ctx.operation}`;
+    return await instrument({
+      tracer: this.tracer,
+      spanName,
+      fn: (span) => {
+        span.setAttribute(NEXUS_SERVICE_ATTR_KEY, input.ctx.service);
+        span.setAttribute(NEXUS_OPERATION_ATTR_KEY, input.ctx.operation);
+        return next(input);
+      },
+      context,
+    });
+  }
+}
+
+/**
+ * Intercepts outbound calls from a Nexus Operation handler.
+ *
+ * Attaches OpenTelemetry context tracing attributes to emitted log messages.
+ *
+ * @experimental This interceptor is experimental and APIs may change without notice.
+ */
+export class OpenTelemetryNexusOutboundInterceptor implements NexusOutboundCallsInterceptor {
+  constructor(protected readonly ctx: nexus.OperationContext) {}
+
+  public getLogAttributes(
+    input: GetLogAttributesInput,
+    next: Next<NexusOutboundCallsInterceptor, 'getLogAttributes'>
+  ): Record<string, unknown> {
+    const span = otel.trace.getSpan(otel.context.active());
+    const spanContext = span?.spanContext();
+    if (spanContext && otel.isSpanContextValid(spanContext)) {
+      return next({
+        trace_id: spanContext.traceId,
+        span_id: spanContext.spanId,
+        trace_flags: `0${spanContext.traceFlags.toString(16)}`,
+        ...input,
+      });
+    } else {
+      return next(input);
+    }
+  }
+
+  public getMetricTags(
+    input: GetMetricTagsInput,
+    next: Next<NexusOutboundCallsInterceptor, 'getMetricTags'>
+  ): GetMetricTagsInput {
+    return next(input);
+  }
+}
+
+/**
+ * Takes an opentelemetry SpanProcessor and turns it into an injected Workflow span exporter sink.
+ */
+export function makeWorkflowExporter(
+  processor: SpanProcessor,
+  resource: Resource
+): InjectedSink<OpenTelemetryWorkflowExporter> {
+  return {
+    export: {
+      fn: (info, spanData) => {
+        const spans = spanData.map((serialized) => {
+          Object.assign(serialized.attributes, info);
+          // Spans are copied over from the isolate and are converted to ReadableSpan instances
+          return extractReadableSpan(serialized, resource);
+        });
+
+        spans.forEach((span) => processor.onEnd(span));
+      },
+    },
+  };
+}
+
+/**
+ * Deserialize a serialized span created by the Workflow isolate
+ */
+function extractReadableSpan(serializable: SerializableSpan, resource: Resource): ReadableSpan {
+  const {
+    spanContext: { traceState, ...restSpanContext },
+    parentSpanContext: serializedParentSpanContext,
+    instrumentationScope,
+    ...rest
+  } = serializable;
+  const spanContext: otel.SpanContext = {
+    // Reconstruct the TraceState from the serialized string.
+    traceState: traceState ? createTraceState(traceState) : undefined,
+    ...restSpanContext,
+  };
+
+  let parentSpanContext: otel.SpanContext | undefined;
+  if (serializedParentSpanContext) {
+    const { traceState: parentTraceState, ...restParentSpanContext } = serializedParentSpanContext;
+    parentSpanContext = {
+      traceState: parentTraceState ? createTraceState(parentTraceState) : undefined,
+      ...restParentSpanContext,
+    };
+  }
+
+  return {
+    spanContext() {
+      return spanContext;
+    },
+    parentSpanContext,
+    instrumentationScope,
+    resource,
+    ...rest,
+  };
+}

package/src/workflow/context-manager.ts

@@ -0,0 +1,53 @@
+import { type AsyncLocalStorage } from 'async_hooks';
+import * as otel from '@opentelemetry/api';
+
+export class ContextManager implements otel.ContextManager {
+  // The workflow sandbox provides AsyncLocalStorage through globalThis.
+  protected storage: AsyncLocalStorage<otel.Context> = new (globalThis as any).AsyncLocalStorage();
+
+  active(): otel.Context {
+    return this.storage.getStore() || otel.ROOT_CONTEXT;
+  }
+
+  bind<T>(context: otel.Context, target: T): T {
+    if (typeof target !== 'function') {
+      throw new TypeError(`Only function binding is supported, got ${typeof target}`);
+    }
+
+    // Stolen from https://github.com/open-telemetry/opentelemetry-js/blob/main/packages/opentelemetry-context-async-hooks/src/AbstractAsyncHooksContextManager.ts
+    const contextWrapper = (...args: unknown[]) => {
+      return this.with(context, () => target.apply(this, args));
+    };
+    Object.defineProperty(contextWrapper, 'length', {
+      enumerable: false,
+      configurable: true,
+      writable: false,
+      value: target.length,
+    });
+    /**
+     * It isn't possible to tell Typescript that contextWrapper is the same as T
+     * so we forced to cast as any here.
+     */
+
+    return contextWrapper as any;
+  }
+
+  enable(): this {
+    return this;
+  }
+
+  disable(): this {
+    this.storage.disable();
+    return this;
+  }
+
+  with<A extends unknown[], F extends (...args: A) => ReturnType<F>>(
+    context: otel.Context,
+    fn: F,
+    thisArg?: ThisParameterType<F>,
+    ...args: A
+  ): ReturnType<F> {
+    const cb = thisArg == null ? fn : fn.bind(thisArg);
+    return this.storage.run(context, cb, ...args);
+  }
+}

package/src/workflow/definitions.ts

@@ -0,0 +1,145 @@
+import type * as otel from '@opentelemetry/api';
+import type * as tracing from '@opentelemetry/sdk-trace-base';
+import type { InstrumentationScope } from '@opentelemetry/core';
+import type { Sink, Sinks } from '@temporalio/workflow';
+
+/**
+ * Serializable version of SpanContext where traceState is converted to a string.
+ */
+export type SerializableSpanContext = Omit<otel.SpanContext, 'traceState'> & { traceState?: string };
+
+/**
+ * Serializable version of the opentelemetry Span for cross isolate copying
+ */
+export interface SerializableSpan {
+  readonly name: string;
+  readonly kind: otel.SpanKind;
+  readonly spanContext: SerializableSpanContext;
+  readonly parentSpanContext?: SerializableSpanContext;
+  readonly startTime: otel.HrTime;
+  readonly endTime: otel.HrTime;
+  readonly status: otel.SpanStatus;
+  readonly attributes: otel.Attributes;
+  readonly links: otel.Link[];
+  readonly events: tracing.TimedEvent[];
+  readonly duration: otel.HrTime;
+  readonly ended: boolean;
+  readonly droppedAttributesCount: number;
+  readonly droppedLinksCount: number;
+  readonly droppedEventsCount: number;
+  // readonly resource: Resource;
+  readonly instrumentationScope: InstrumentationScope;
+}
+
+export interface OpenTelemetryWorkflowExporter extends Sink {
+  export(span: SerializableSpan[]): void;
+}
+
+/**
+ * Required external dependencies for Workflow interceptor to export spans
+ */
+export interface OpenTelemetrySinks extends Sinks {
+  exporter: OpenTelemetryWorkflowExporter;
+}
+
+export enum SpanName {
+  /**
+   * Workflow is scheduled by a client
+   */
+  WORKFLOW_START = 'StartWorkflow',
+
+  /**
+   * Workflow is signalled
+   */
+  WORKFLOW_SIGNAL = 'SignalWorkflow',
+
+  /**
+   * Workflow is client calls signalWithStart
+   */
+  WORKFLOW_SIGNAL_WITH_START = 'SignalWithStartWorkflow',
+
+  /**
+   * Workflow is queried
+   */
+  WORKFLOW_QUERY = 'QueryWorkflow',
+
+  /**
+   * Workflow update is started by client
+   */
+  WORKFLOW_START_UPDATE = 'StartWorkflowUpdate',
+
+  /**
+   * Workflow is started with an update
+   */
+  WORKFLOW_UPDATE_WITH_START = 'UpdateWithStartWorkflow',
+
+  /**
+   * Workflow handles an incoming signal
+   */
+  WORKFLOW_HANDLE_SIGNAL = 'HandleSignal',
+
+  /**
+   * Workflow handles an incoming query
+   */
+  WORKFLOW_HANDLE_QUERY = 'HandleQuery',
+
+  /**
+   * Workflow handles an incoming update
+   */
+  WORKFLOW_HANDLE_UPDATE = 'HandleUpdate',
+
+  /**
+   * Workflow validates an incoming update
+   */
+  WORKFLOW_VALIDATE_UPDATE = 'ValidateUpdate',
+
+  /**
+   * Workflow is terminated
+   */
+  WORKFLOW_TERMINATE = 'TerminateWorkflow',
+
+  /**
+   * Workflow is cancelled
+   */
+  WORKFLOW_CANCEL = 'CancelWorkflow',
+
+  /**
+   * Workflow is described
+   */
+  WORKFLOW_DESCRIBE = 'DescribeWorkflow',
+
+  /**
+   * Workflow run is executing
+   */
+  WORKFLOW_EXECUTE = 'RunWorkflow',
+  /**
+   * Child Workflow is started (by parent Workflow)
+   */
+  CHILD_WORKFLOW_START = 'StartChildWorkflow',
+  /**
+   * Activity is scheduled by a Workflow
+   */
+  ACTIVITY_START = 'StartActivity',
+  /**
+   * Activity is executing
+   */
+  ACTIVITY_EXECUTE = 'RunActivity',
+  /**
+   * Workflow is continuing as new
+   */
+  CONTINUE_AS_NEW = 'ContinueAsNew',
+  /**
+   * Nexus operation is started
+   */
+  NEXUS_OPERATION_START = 'StartNexusOperation',
+  /**
+   * Nexus operation is executing
+   */
+  NEXUS_START_OPERATION_EXECUTE = 'RunStartNexusOperation',
+  /**
+   * Nexus operation cancel is executing
+   */
+  NEXUS_CANCEL_OPERATION_EXECUTE = 'RunCancelNexusOperation',
+}
+
+export const SPAN_DELIMITER = ':';

package/src/workflow/id-generator.ts

@@ -0,0 +1,30 @@
+import type { IdGenerator } from '@opentelemetry/sdk-trace-base';
+import { getRandomStream } from './workflow-imports';
+
+const OTEL_ID_RANDOM_STREAM = '@temporalio/interceptors-opentelemetry-v2/id-generator';
+
+function randomHex(length: number): string {
+  const stream = getRandomStream(OTEL_ID_RANDOM_STREAM);
+  const chars: string[] = [];
+  for (let i = 0; i < length; i++) {
+    const nibble = (stream.random() * 16) >>> 0;
+    chars.push(nibble.toString(16));
+  }
+  return chars.join('');
+}
+
+/**
+ * Generates span and trace IDs from a named workflow random stream.
+ */
+export class DeterministicIdGenerator implements IdGenerator {
+  generateTraceId(): string {
+    const id = randomHex(32);
+    // Ensure non-zero per W3C Trace Context spec
+    return id === '00000000000000000000000000000000' ? '00000000000000000000000000000001' : id;
+  }
+
+  generateSpanId(): string {
+    const id = randomHex(16);
+    return id === '0000000000000000' ? '0000000000000001' : id;
+  }
+}