@temporalio/interceptors-opentelemetry-v2 1.19.0

package/src/client/index.ts

@@ -0,0 +1,220 @@
+import * as otel from '@opentelemetry/api';
+import type {
+  Next,
+  WorkflowSignalInput,
+  WorkflowSignalWithStartInput,
+  WorkflowStartInput,
+  WorkflowStartOutput,
+  WorkflowStartUpdateInput,
+  WorkflowStartUpdateOutput,
+  WorkflowStartUpdateWithStartInput,
+  WorkflowStartUpdateWithStartOutput,
+  WorkflowQueryInput,
+  WorkflowTerminateInput,
+  WorkflowCancelInput,
+  WorkflowDescribeInput,
+  WorkflowClientInterceptor,
+  TerminateWorkflowExecutionResponse,
+  RequestCancelWorkflowExecutionResponse,
+  DescribeWorkflowExecutionResponse,
+} from '@temporalio/client';
+import {
+  instrument,
+  headersWithContext,
+  RUN_ID_ATTR_KEY,
+  WORKFLOW_ID_ATTR_KEY,
+  UPDATE_ID_ATTR_KEY,
+  TERMINATE_REASON_ATTR_KEY,
+} from '../instrumentation';
+import { SpanName, SPAN_DELIMITER } from '../workflow/definitions';
+
+export interface InterceptorOptions {
+  readonly tracer?: otel.Tracer;
+}
+
+/**
+ * Intercepts calls to start a Workflow.
+ *
+ * Wraps the operation in an opentelemetry Span and passes it to the Workflow via headers.
+ *
+ * @experimental This interceptor is experimental and APIs may change without notice.
+ */
+export class OpenTelemetryWorkflowClientInterceptor implements WorkflowClientInterceptor {
+  protected readonly tracer: otel.Tracer;
+
+  constructor(options?: InterceptorOptions) {
+    this.tracer = options?.tracer ?? otel.trace.getTracer('@temporalio/interceptor-client');
+  }
+
+  async start(input: WorkflowStartInput, next: Next<WorkflowClientInterceptor, 'start'>): Promise<string> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: `${SpanName.WORKFLOW_START}${SPAN_DELIMITER}${input.workflowType}`,
+      fn: async (span) => {
+        const headers = headersWithContext(input.headers);
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.options.workflowId);
+        const runId = await next({ ...input, headers });
+        span.setAttribute(RUN_ID_ATTR_KEY, runId);
+        return runId;
+      },
+    });
+  }
+
+  async signal(input: WorkflowSignalInput, next: Next<WorkflowClientInterceptor, 'signal'>): Promise<void> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: `${SpanName.WORKFLOW_SIGNAL}${SPAN_DELIMITER}${input.signalName}`,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId);
+        const headers = headersWithContext(input.headers);
+        await next({ ...input, headers });
+      },
+    });
+  }
+
+  async startWithDetails(
+    input: WorkflowStartInput,
+    next: Next<WorkflowClientInterceptor, 'startWithDetails'>
+  ): Promise<WorkflowStartOutput> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: `${SpanName.WORKFLOW_START}${SPAN_DELIMITER}${input.workflowType}`,
+      fn: async (span) => {
+        const headers = headersWithContext(input.headers);
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.options.workflowId);
+        const output = await next({ ...input, headers });
+        span.setAttribute(RUN_ID_ATTR_KEY, output.runId);
+        return output;
+      },
+    });
+  }
+
+  async startUpdate(
+    input: WorkflowStartUpdateInput,
+    next: Next<WorkflowClientInterceptor, 'startUpdate'>
+  ): Promise<WorkflowStartUpdateOutput> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: `${SpanName.WORKFLOW_START_UPDATE}${SPAN_DELIMITER}${input.updateName}`,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId);
+        if (input.options.updateId) {
+          span.setAttribute(UPDATE_ID_ATTR_KEY, input.options.updateId);
+        }
+        const headers = headersWithContext(input.headers);
+        const output = await next({ ...input, headers });
+        span.setAttribute(RUN_ID_ATTR_KEY, output.workflowRunId);
+        return output;
+      },
+    });
+  }
+
+  async startUpdateWithStart(
+    input: WorkflowStartUpdateWithStartInput,
+    next: Next<WorkflowClientInterceptor, 'startUpdateWithStart'>
+  ): Promise<WorkflowStartUpdateWithStartOutput> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: `${SpanName.WORKFLOW_UPDATE_WITH_START}${SPAN_DELIMITER}${input.updateName}`,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowStartOptions.workflowId);
+        if (input.updateOptions.updateId) {
+          span.setAttribute(UPDATE_ID_ATTR_KEY, input.updateOptions.updateId);
+        }
+        const workflowStartHeaders = headersWithContext(input.workflowStartHeaders);
+        const updateHeaders = headersWithContext(input.updateHeaders);
+        const output = await next({ ...input, workflowStartHeaders, updateHeaders });
+        if (output.workflowExecution.runId) {
+          span.setAttribute(RUN_ID_ATTR_KEY, output.workflowExecution.runId);
+        }
+        return output;
+      },
+    });
+  }
+
+  async signalWithStart(
+    input: WorkflowSignalWithStartInput,
+    next: Next<WorkflowClientInterceptor, 'signalWithStart'>
+  ): Promise<string> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: `${SpanName.WORKFLOW_SIGNAL_WITH_START}${SPAN_DELIMITER}${input.workflowType}`,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.options.workflowId);
+        const headers = headersWithContext(input.headers);
+        const runId = await next({ ...input, headers });
+        span.setAttribute(RUN_ID_ATTR_KEY, runId);
+        return runId;
+      },
+    });
+  }
+
+  async query(input: WorkflowQueryInput, next: Next<WorkflowClientInterceptor, 'query'>): Promise<unknown> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: `${SpanName.WORKFLOW_QUERY}${SPAN_DELIMITER}${input.queryType}`,
+      fn: async (span) => {
+        const headers = headersWithContext(input.headers);
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId);
+        if (input.workflowExecution.runId) {
+          span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId);
+        }
+        return await next({ ...input, headers });
+      },
+    });
+  }
+
+  async terminate(
+    input: WorkflowTerminateInput,
+    next: Next<WorkflowClientInterceptor, 'terminate'>
+  ): Promise<TerminateWorkflowExecutionResponse> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: SpanName.WORKFLOW_TERMINATE,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId);
+        if (input.workflowExecution.runId) {
+          span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId);
+        }
+        if (input.reason) {
+          span.setAttribute(TERMINATE_REASON_ATTR_KEY, input.reason);
+        }
+        return await next(input);
+      },
+    });
+  }
+
+  async cancel(
+    input: WorkflowCancelInput,
+    next: Next<WorkflowClientInterceptor, 'cancel'>
+  ): Promise<RequestCancelWorkflowExecutionResponse> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: SpanName.WORKFLOW_CANCEL,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId);
+        if (input.workflowExecution.runId) {
+          span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId);
+        }
+        return await next(input);
+      },
+    });
+  }
+
+  async describe(
+    input: WorkflowDescribeInput,
+    next: Next<WorkflowClientInterceptor, 'describe'>
+  ): Promise<DescribeWorkflowExecutionResponse> {
+    return await instrument({
+      tracer: this.tracer,
+      spanName: SpanName.WORKFLOW_DESCRIBE,
+      fn: async (span) => {
+        span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId);
+        if (input.workflowExecution.runId) {
+          span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId);
+        }
+        return await next(input);
+      },
+    });
+  }
+}

package/src/index.ts

@@ -0,0 +1,14 @@
+/**
+ * `npm i @temporalio/interceptors-opentelemetry-v2`
+ *
+ * Interceptors that add OpenTelemetry tracing.
+ *
+ * [Documentation](https://docs.temporal.io/typescript/logging#opentelemetry-tracing)
+ *
+ * @module
+ */
+
+export * from './plugin';
+export * from './workflow';
+export * from './worker';
+export { OpenTelemetryWorkflowClientInterceptor } from './client';

package/src/instrumentation.ts

@@ -0,0 +1,159 @@
+/**
+ * opentelemetry instrumentation helper functions
+ * @module
+ */
+import * as otel from '@opentelemetry/api';
+import {
+  type Headers,
+  ApplicationFailure,
+  ApplicationFailureCategory,
+  defaultPayloadConverter,
+} from '@temporalio/common';
+
+/** Default trace header for opentelemetry interceptors */
+export const TRACE_HEADER = '_tracer-data';
+/** As in workflow run id */
+export const RUN_ID_ATTR_KEY = 'run_id';
+/** As in workflow id */
+export const WORKFLOW_ID_ATTR_KEY = 'temporalWorkflowId';
+/** As in activity id */
+export const ACTIVITY_ID_ATTR_KEY = 'temporalActivityId';
+/** As in update id */
+export const UPDATE_ID_ATTR_KEY = 'temporalUpdateId';
+/** As in termination reason */
+export const TERMINATE_REASON_ATTR_KEY = 'temporalTerminateReason';
+/** As in Nexus service */
+export const NEXUS_SERVICE_ATTR_KEY = 'temporalNexusService';
+/** As in Nexus operation */
+export const NEXUS_OPERATION_ATTR_KEY = 'temporalNexusOperation';
+/** As in Nexus endpoint */
+export const NEXUS_ENDPOINT_ATTR_KEY = 'temporalNexusEndpoint';
+
+const payloadConverter = defaultPayloadConverter;
+
+/**
+ * If found, return an otel Context deserialized from the provided headers
+ */
+export function extractContextFromHeaders(headers: Headers): otel.Context | undefined {
+  const encodedSpanContext = headers[TRACE_HEADER];
+  if (encodedSpanContext === undefined) {
+    return undefined;
+  }
+  const textMap: Record<string, string> = payloadConverter.fromPayload(encodedSpanContext);
+  return otel.propagation.extract(otel.context.active(), textMap, otel.defaultTextMapGetter);
+}
+
+/**
+ * If found, return an otel Context deserialized from the provided Nexus headers.
+ *
+ * Nexus headers are plain `Record<string, string>`, so we extract the context
+ * directly from the string map.
+ */
+export function extractContextFromNexusHeaders(headers: Record<string, string>): otel.Context | undefined {
+  if (Object.keys(headers).length === 0) {
+    return undefined;
+  }
+  return otel.propagation.extract(otel.context.active(), headers, otel.defaultTextMapGetter);
+}
+
+/**
+ * Given headers, return new headers with the current otel context inserted
+ */
+export function headersWithContext(headers: Headers): Headers {
+  const carrier = {};
+  otel.propagation.inject(otel.context.active(), carrier, otel.defaultTextMapSetter);
+  return { ...headers, [TRACE_HEADER]: payloadConverter.toPayload(carrier) };
+}
+
+/**
+ * Given Nexus headers, return new headers with the current otel context injected directly
+ */
+export function nexusHeadersWithContext(headers: Record<string, string>): Record<string, string> {
+  const carrier: Record<string, string> = {};
+  otel.propagation.inject(otel.context.active(), carrier, otel.defaultTextMapSetter);
+  return { ...headers, ...carrier };
+}
+
+async function wrapWithSpan<T>(
+  span: otel.Span,
+  fn: (span: otel.Span) => Promise<T>,
+  acceptableErrors?: (err: unknown) => boolean
+): Promise<T> {
+  try {
+    const ret = await fn(span);
+    span.setStatus({ code: otel.SpanStatusCode.OK });
+    return ret;
+  } catch (err: any) {
+    maybeAddErrorToSpan(err, span, acceptableErrors);
+    throw err;
+  } finally {
+    span.end();
+  }
+}
+
+function wrapWithSpanSync<T>(
+  span: otel.Span,
+  fn: (span: otel.Span) => T,
+  acceptableErrors?: (err: unknown) => boolean
+): T {
+  try {
+    const ret = fn(span);
+    span.setStatus({ code: otel.SpanStatusCode.OK });
+    return ret;
+  } catch (err: any) {
+    maybeAddErrorToSpan(err, span, acceptableErrors);
+    throw err;
+  } finally {
+    span.end();
+  }
+}
+
+function maybeAddErrorToSpan(err: any, span: otel.Span, acceptableErrors?: (err: unknown) => boolean): void {
+  const isBenignErr = err instanceof ApplicationFailure && err.category === ApplicationFailureCategory.BENIGN;
+  if (acceptableErrors === undefined || !acceptableErrors(err)) {
+    const statusCode = isBenignErr ? otel.SpanStatusCode.UNSET : otel.SpanStatusCode.ERROR;
+    const message = err != null && typeof err.message === 'string' ? err.message : String(err);
+    span.setStatus({ code: statusCode, message });
+    span.recordException(err);
+  } else {
+    span.setStatus({ code: otel.SpanStatusCode.OK });
+  }
+}
+
+export interface InstrumentOptions<T> {
+  tracer: otel.Tracer;
+  spanName: string;
+  fn: (span: otel.Span) => Promise<T>;
+  context?: otel.Context;
+  acceptableErrors?: (err: unknown) => boolean;
+}
+
+export type InstrumentOptionsSync<T> = Omit<InstrumentOptions<T>, 'fn'> & { fn: (span: otel.Span) => T };
+
+/**
+ * Wraps `fn` in a span which ends when function returns or throws
+ */
+export async function instrument<T>({
+  tracer,
+  spanName,
+  fn,
+  context,
+  acceptableErrors,
+}: InstrumentOptions<T>): Promise<T> {
+  if (context) {
+    return await tracer.startActiveSpan(
+      spanName,
+      {},
+      context,
+      async (span) => await wrapWithSpan(span, fn, acceptableErrors)
+    );
+  }
+  return await tracer.startActiveSpan(spanName, async (span) => await wrapWithSpan(span, fn, acceptableErrors));
+}
+
+export function instrumentSync<T>({ tracer, spanName, fn, context, acceptableErrors }: InstrumentOptionsSync<T>): T {
+  if (context) {
+    return tracer.startActiveSpan(spanName, {}, context, (span) => wrapWithSpanSync(span, fn, acceptableErrors));
+  }
+  return tracer.startActiveSpan(spanName, (span) => wrapWithSpanSync(span, fn, acceptableErrors));
+}

package/src/plugin.ts

@@ -0,0 +1,86 @@
+import type { SpanProcessor } from '@opentelemetry/sdk-trace-base';
+import type { Resource } from '@opentelemetry/resources';
+import { SimplePlugin } from '@temporalio/plugin';
+import type { InjectedSinks, ReplayWorkerOptions, WorkerOptions } from '@temporalio/worker';
+import type { InterceptorOptions } from './client';
+import { OpenTelemetryWorkflowClientInterceptor } from './client';
+import {
+  makeWorkflowExporter,
+  OpenTelemetryActivityInboundInterceptor,
+  OpenTelemetryActivityOutboundInterceptor,
+  OpenTelemetryNexusInboundInterceptor,
+  OpenTelemetryNexusOutboundInterceptor,
+} from './worker';
+import type { OpenTelemetrySinks } from './workflow';
+
+/**
+ * Configuration options for {@link OpenTelemetryPlugin}.
+ *
+ * @experimental Plugins is an experimental feature; APIs may change without notice.
+ */
+export interface OpenTelemetryPluginOptions extends InterceptorOptions {
+  /** OpenTelemetry resource attributes to attach to exported spans */
+  readonly resource: Resource;
+  /** Exporter used to send spans to a tracing backend */
+  readonly spanProcessor: SpanProcessor;
+}
+
+/**
+ * A plugin that adds OpenTelemetry tracing.
+ *
+ * Configures Client, Activity, and Workflow interceptors for trace propagation and injects
+ * a span exporter sink for Workflow spans.
+ *
+ * @experimental Plugins is an experimental feature; APIs may change without notice.
+ */
+export class OpenTelemetryPlugin extends SimplePlugin {
+  constructor(readonly otelOptions: OpenTelemetryPluginOptions) {
+    const workflowInterceptorsPath = require.resolve('./workflow-interceptors');
+    const interceptorOptions = otelOptions.tracer ? { tracer: otelOptions.tracer } : {};
+    super({
+      name: 'OpenTelemetryPlugin',
+      clientInterceptors: {
+        workflow: [new OpenTelemetryWorkflowClientInterceptor(interceptorOptions)],
+      },
+      workerInterceptors: {
+        client: {
+          workflow: [new OpenTelemetryWorkflowClientInterceptor(interceptorOptions)],
+        },
+        workflowModules: [workflowInterceptorsPath],
+        activity: [
+          (ctx) => ({
+            inbound: new OpenTelemetryActivityInboundInterceptor(ctx, interceptorOptions),
+            outbound: new OpenTelemetryActivityOutboundInterceptor(ctx),
+          }),
+        ],
+        nexus: [
+          (ctx) => ({
+            inbound: new OpenTelemetryNexusInboundInterceptor(ctx, interceptorOptions),
+            outbound: new OpenTelemetryNexusOutboundInterceptor(ctx),
+          }),
+        ],
+      },
+    });
+  }
+
+  configureWorker(options: WorkerOptions): WorkerOptions {
+    return super.configureWorker(this.injectSinks(options));
+  }
+
+  configureReplayWorker(options: ReplayWorkerOptions): ReplayWorkerOptions {
+    return super.configureReplayWorker(this.injectSinks(options));
+  }
+
+  private injectSinks<T extends { sinks?: InjectedSinks<any> }>(options: T): T {
+    const sinks: InjectedSinks<OpenTelemetrySinks> = {
+      exporter: makeWorkflowExporter(this.otelOptions.spanProcessor, this.otelOptions.resource),
+    };
+    return {
+      ...options,
+      sinks: {
+        ...options.sinks,
+        ...sinks,
+      },
+    };
+  }
+}