@aztec/telemetry-client 0.0.1-commit.b655e406 → 0.0.1-commit.b8a057fa

package/src/monitored_batch_span_processor.ts

@@ -0,0 +1,100 @@
+import type { Logger } from '@aztec/foundation/log';
+
+import { type Context, SpanStatusCode } from '@opentelemetry/api';
+import { hrTimeToMilliseconds } from '@opentelemetry/core';
+import type { SpanExporter } from '@opentelemetry/sdk-trace-base';
+import { BatchSpanProcessor, type BufferConfig, type ReadableSpan, type Span } from '@opentelemetry/sdk-trace-node';
+
+/** Minimum interval between drop warnings to avoid log spam. */
+const DROP_WARNING_INTERVAL_MS = 30_000;
+
+const DEFAULT_MIN_TRACE_DURATION_MS = 10;
+
+const DEFAULT_MAX_QUEUE_SIZE = 16384;
+
+/** Cap on the per-export batch size, so a large queue can actually be drained instead of dribbling out
+ * at the SDK default of 512 spans per scheduled export. Kept <= maxQueueSize per the BatchSpanProcessor contract. */
+const DEFAULT_MAX_EXPORT_BATCH_SIZE = 2048;
+
+export type MonitoredBatchSpanProcessorConfig = BufferConfig & {
+  minTraceDurationMs?: number;
+};
+
+/**
+ * Wraps BatchSpanProcessor to emit warnings when spans are dropped due to a full queue.
+ * The standard BatchSpanProcessor silently discards spans when its internal queue reaches
+ * maxQueueSize, making telemetry data loss invisible to operators.
+ */
+export class MonitoredBatchSpanProcessor extends BatchSpanProcessor {
+  private readonly maxQueueSize: number;
+  private readonly minTraceDurationMs: number;
+  private readonly log: Logger;
+
+  private approxQueueSize = 0;
+  private droppedSinceLastWarning = 0;
+  private totalDropped = 0;
+  private lastWarningTime = 0;
+
+  constructor(exporter: SpanExporter, log: Logger, config?: MonitoredBatchSpanProcessorConfig) {
+    const maxQueueSize = config?.maxQueueSize ?? DEFAULT_MAX_QUEUE_SIZE;
+    const maxExportBatchSize = Math.min(config?.maxExportBatchSize ?? DEFAULT_MAX_EXPORT_BATCH_SIZE, maxQueueSize);
+    super(exporter, { ...config, maxQueueSize, maxExportBatchSize });
+    this.maxQueueSize = maxQueueSize;
+    this.minTraceDurationMs = Math.max(0, config?.minTraceDurationMs ?? DEFAULT_MIN_TRACE_DURATION_MS);
+    this.log = log;
+  }
+
+  override onStart(span: Span, parentContext: Context): void {
+    super.onStart(span, parentContext);
+  }
+
+  override onEnd(span: ReadableSpan): void {
+    if (this.shouldDropShortSpan(span)) {
+      return;
+    }
+
+    if (this.approxQueueSize >= this.maxQueueSize) {
+      this.droppedSinceLastWarning++;
+      this.totalDropped++;
+      this.maybeLogDropWarning();
+    } else {
+      this.approxQueueSize++;
+    }
+    super.onEnd(span);
+  }
+
+  override async forceFlush(): Promise<void> {
+    await super.forceFlush();
+    this.approxQueueSize = 0;
+  }
+
+  override async shutdown(): Promise<void> {
+    if (this.totalDropped > 0) {
+      this.log.warn(`BatchSpanProcessor shutting down with ${this.totalDropped} total spans dropped`, {
+        totalDropped: this.totalDropped,
+      });
+    }
+    await super.shutdown();
+  }
+
+  private shouldDropShortSpan(span: ReadableSpan): boolean {
+    return (
+      this.minTraceDurationMs > 0 &&
+      span.status.code !== SpanStatusCode.ERROR &&
+      hrTimeToMilliseconds(span.duration) < this.minTraceDurationMs
+    );
+  }
+
+  private maybeLogDropWarning(): void {
+    const now = Date.now();
+    if (now - this.lastWarningTime >= DROP_WARNING_INTERVAL_MS) {
+      this.log.warn(
+        `BatchSpanProcessor dropping spans: queue full (maxQueueSize=${this.maxQueueSize}). ` +
+          `${this.droppedSinceLastWarning} dropped since last warning, ${this.totalDropped} total.`,
+        { droppedSinceLastWarning: this.droppedSinceLastWarning, totalDropped: this.totalDropped },
+      );
+      this.droppedSinceLastWarning = 0;
+      this.lastWarningTime = now;
+    }
+  }
+}

package/src/nodejs_metrics_monitor.ts

@@ -2,18 +2,11 @@ import type { Observable } from '@opentelemetry/api';
 import { type EventLoopUtilization, type IntervalHistogram, monitorEventLoopDelay, performance } from 'node:perf_hooks';
 
 import * as Attributes from './attributes.js';
+import { createUpDownCounterWithDefault } from './metric-utils.js';
 import * as Metrics from './metrics.js';
-import {
-  type BatchObservableResult,
-  type Meter,
-  type ObservableGauge,
-  type UpDownCounter,
-  ValueType,
-} from './telemetry.js';
-
-/**
- * Detector for custom Aztec attributes
- */
+import type { BatchObservableResult, Meter, ObservableGauge, UpDownCounter } from './telemetry.js';
+
+/** Monitors Node.js runtime metrics */
 export class NodejsMetricsMonitor {
   private eventLoopDelayGauges: {
     min: ObservableGauge;
@@ -38,53 +31,27 @@ export class NodejsMetricsMonitor {
   private eventLoopDelay: IntervalHistogram;
 
   constructor(private meter: Meter) {
-    const nsObsGauge = (name: (typeof Metrics)[keyof typeof Metrics], description: string) =>
-      meter.createObservableGauge(name, {
-        unit: 'ns',
-        valueType: ValueType.INT,
-        description,
-      });
-
     this.eventLoopDelayGauges = {
-      min: nsObsGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MIN, 'Minimum delay of the event loop'),
-      mean: nsObsGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MEAN, 'Mean delay of the event loop'),
-      max: nsObsGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MAX, 'Max delay of the event loop'),
-      stddev: nsObsGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_STDDEV, 'Stddev delay of the event loop'),
-      p50: nsObsGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P50, 'P50 delay of the event loop'),
-      p90: nsObsGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P90, 'P90 delay of the event loop'),
-      p99: nsObsGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P99, 'P99 delay of the event loop'),
+      min: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MIN),
+      mean: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MEAN),
+      max: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MAX),
+      stddev: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_STDDEV),
+      p50: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P50),
+      p90: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P90),
+      p99: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P99),
     };
 
-    this.eventLoopUilization = meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_UTILIZATION, {
-      valueType: ValueType.DOUBLE,
-      description: 'How busy is the event loop',
+    this.eventLoopUilization = meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_UTILIZATION);
+    this.eventLoopTime = createUpDownCounterWithDefault(meter, Metrics.NODEJS_EVENT_LOOP_TIME, {
+      [Attributes.NODEJS_EVENT_LOOP_STATE]: ['idle', 'active'],
     });
-
-    this.eventLoopTime = meter.createUpDownCounter(Metrics.NODEJS_EVENT_LOOP_TIME, {
-      unit: 'ms',
-      valueType: ValueType.INT,
-      description: 'How much time the event loop has spent in a given state',
-    });
-
     this.eventLoopDelay = monitorEventLoopDelay();
 
     this.memoryGauges = {
-      heapUsed: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_USAGE, {
-        unit: 'By',
-        description: 'Memory used by the V8 heap',
-      }),
-      heapTotal: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_TOTAL, {
-        unit: 'By',
-        description: 'The max size the V8 heap can grow to',
-      }),
-      arrayBuffers: meter.createObservableGauge(Metrics.NODEJS_MEMORY_BUFFER_USAGE, {
-        unit: 'By',
-        description: 'Memory allocated for buffers (includes native memory used)',
-      }),
-      external: meter.createObservableGauge(Metrics.NODEJS_MEMORY_NATIVE_USAGE, {
-        unit: 'By',
-        description: 'Memory allocated for native C++ objects',
-      }),
+      heapUsed: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_USAGE),
+      heapTotal: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_TOTAL),
+      arrayBuffers: meter.createObservableGauge(Metrics.NODEJS_MEMORY_BUFFER_USAGE),
+      external: meter.createObservableGauge(Metrics.NODEJS_MEMORY_NATIVE_USAGE),
     };
   }
 

package/src/noop.ts

@@ -1,13 +1,63 @@
-import { type Meter, type Span, type SpanContext, type Tracer, createNoopMeter } from '@opentelemetry/api';
+import { type Context, type Span, type SpanContext, type Tracer, createNoopMeter } from '@opentelemetry/api';
 
-import type { TelemetryClient } from './telemetry.js';
+import type { MetricDefinition } from './metrics.js';
+import type {
+  Gauge,
+  Histogram,
+  Meter,
+  ObservableGauge,
+  ObservableUpDownCounter,
+  TelemetryClient,
+  UpDownCounter,
+} from './telemetry.js';
+
+/** A no-op meter that implements our custom Meter interface */
+class NoopMeter implements Meter {
+  private otelMeter = createNoopMeter();
+
+  createGauge(_metric: MetricDefinition): Gauge {
+    return this.otelMeter.createGauge('');
+  }
+
+  createObservableGauge(_metric: MetricDefinition): ObservableGauge {
+    return this.otelMeter.createObservableGauge('');
+  }
+
+  createHistogram(_metric: MetricDefinition, _extraOptions?: Parameters<Meter['createHistogram']>[1]): Histogram {
+    return this.otelMeter.createHistogram('');
+  }
+
+  createUpDownCounter(_metric: MetricDefinition): UpDownCounter {
+    return this.otelMeter.createUpDownCounter('');
+  }
+
+  createObservableUpDownCounter(_metric: MetricDefinition): ObservableUpDownCounter {
+    return this.otelMeter.createObservableUpDownCounter('');
+  }
+
+  addBatchObservableCallback(
+    callback: Parameters<Meter['addBatchObservableCallback']>[0],
+    observables: Parameters<Meter['addBatchObservableCallback']>[1],
+  ): void {
+    this.otelMeter.addBatchObservableCallback(callback, observables);
+  }
+
+  removeBatchObservableCallback(
+    callback: Parameters<Meter['removeBatchObservableCallback']>[0],
+    observables: Parameters<Meter['removeBatchObservableCallback']>[1],
+  ): void {
+    this.otelMeter.removeBatchObservableCallback(callback, observables);
+  }
+}
 
 export class NoopTelemetryClient implements TelemetryClient {
+  private meter = new NoopMeter();
+
   setExportedPublicTelemetry(_prefixes: string[]): void {}
   setPublicTelemetryCollectFrom(_roles: string[]): void {}
 
   getMeter(): Meter {
-    return createNoopMeter();
+    return this.meter;
   }
 
   getTracer(): Tracer {
@@ -25,6 +75,14 @@ export class NoopTelemetryClient implements TelemetryClient {
   isEnabled() {
     return false;
   }
+
+  getTraceContext(): string | undefined {
+    return undefined;
+  }
+
+  extractPropagatedContext(_traceContext: string): Context | undefined {
+    return undefined;
+  }
 }
 
 // @opentelemetry/api internally uses NoopTracer and NoopSpan but they're not exported

package/src/otel.ts

@@ -1,16 +1,20 @@
 import { type LogData, type Logger, addLogDataHandler } from '@aztec/foundation/log';
 
 import {
+  type Context,
   DiagConsoleLogger,
   DiagLogLevel,
-  type Meter,
+  type Meter as OtelMeter,
+  ROOT_CONTEXT,
   type Tracer,
   type TracerProvider,
   context,
   diag,
   isSpanContextValid,
+  propagation,
   trace,
 } from '@opentelemetry/api';
+import { W3CTraceContextPropagator } from '@opentelemetry/core';
 import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
 import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
 import { HostMetrics } from '@opentelemetry/host-metrics';
@@ -24,24 +28,79 @@ import {
   type PeriodicExportingMetricReaderOptions,
   View,
 } from '@opentelemetry/sdk-metrics';
-import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
 import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
 
 import type { TelemetryClientConfig } from './config.js';
+import { toMetricOptions } from './metric-utils.js';
+import type { MetricDefinition } from './metrics.js';
+import { MonitoredBatchSpanProcessor } from './monitored_batch_span_processor.js';
 import { NodejsMetricsMonitor } from './nodejs_metrics_monitor.js';
 import { OtelFilterMetricExporter, PublicOtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
 import { registerOtelLoggerProvider } from './otel_logger_provider.js';
 import { getOtelResource } from './otel_resource.js';
-import type { TelemetryClient } from './telemetry.js';
+import type {
+  Gauge,
+  Histogram,
+  Meter,
+  ObservableGauge,
+  ObservableUpDownCounter,
+  TelemetryClient,
+  UpDownCounter,
+} from './telemetry.js';
+
+/** Wraps an OpenTelemetry Meter to implement our custom Meter interface */
+class WrappedMeter implements Meter {
+  constructor(private otelMeter: OtelMeter) {}
+
+  createGauge(metric: MetricDefinition): Gauge {
+    return this.otelMeter.createGauge(metric.name, toMetricOptions(metric));
+  }
+
+  createObservableGauge(metric: MetricDefinition): ObservableGauge {
+    return this.otelMeter.createObservableGauge(metric.name, toMetricOptions(metric));
+  }
+
+  createHistogram(metric: MetricDefinition, extraOptions?: Parameters<Meter['createHistogram']>[1]): Histogram {
+    return this.otelMeter.createHistogram(metric.name, { ...toMetricOptions(metric), ...extraOptions });
+  }
+
+  createUpDownCounter(metric: MetricDefinition): UpDownCounter {
+    return this.otelMeter.createUpDownCounter(metric.name, toMetricOptions(metric));
+  }
+
+  createObservableUpDownCounter(metric: MetricDefinition): ObservableUpDownCounter {
+    return this.otelMeter.createObservableUpDownCounter(metric.name, toMetricOptions(metric));
+  }
+
+  addBatchObservableCallback(
+    callback: Parameters<Meter['addBatchObservableCallback']>[0],
+    observables: Parameters<Meter['addBatchObservableCallback']>[1],
+  ): void {
+    this.otelMeter.addBatchObservableCallback(callback, observables);
+  }
+
+  removeBatchObservableCallback(
+    callback: Parameters<Meter['removeBatchObservableCallback']>[0],
+    observables: Parameters<Meter['removeBatchObservableCallback']>[1],
+  ): void {
+    this.otelMeter.removeBatchObservableCallback(callback, observables);
+  }
+}
 
 export type OpenTelemetryClientFactory = (resource: IResource, log: Logger) => OpenTelemetryClient;
 
 export class OpenTelemetryClient implements TelemetryClient {
   hostMetrics: HostMetrics | undefined;
   nodejsMetricsMonitor: NodejsMetricsMonitor | undefined;
-  private meters: Map<string, Meter> = new Map<string, Meter>();
+  private meters: Map<string, WrappedMeter> = new Map<string, WrappedMeter>();
   private tracers: Map<string, Tracer> = new Map<string, Tracer>();
 
+  /** Memoized shutdown promise. The telemetry client is shared between the aztec-node and an embedded prover-node,
+   * so stop() can be invoked more than once; the providers throw "shutdown may only be called once" and
+   * "invalid attempt to force flush after shutdown" if that happens. Guarding here makes stop()/flush() idempotent. */
+  private stopPromise: Promise<void> | undefined;
+
   protected constructor(
     private resource: IResource,
     private meterProvider: MeterProvider,
@@ -62,7 +121,8 @@ export class OpenTelemetryClient implements TelemetryClient {
   getMeter(name: string): Meter {
     let meter = this.meters.get(name);
     if (!meter) {
-      meter = this.meterProvider.getMeter(name, this.resource.attributes[ATTR_SERVICE_VERSION] as string);
+      const otelMeter = this.meterProvider.getMeter(name, this.resource.attributes[ATTR_SERVICE_VERSION] as string);
+      meter = new WrappedMeter(otelMeter);
       this.meters.set(name, meter);
     }
     return meter;
@@ -100,9 +160,10 @@ export class OpenTelemetryClient implements TelemetryClient {
       meterProvider: this.meterProvider,
     });
 
-    this.nodejsMetricsMonitor = new NodejsMetricsMonitor(
+    const nodejsMeter = new WrappedMeter(
       this.meterProvider.getMeter(this.resource.attributes[ATTR_SERVICE_NAME] as string),
     );
+    this.nodejsMetricsMonitor = new NodejsMetricsMonitor(nodejsMeter);
 
     this.hostMetrics.start();
     this.nodejsMetricsMonitor.start();
@@ -113,6 +174,10 @@ export class OpenTelemetryClient implements TelemetryClient {
   }
 
   public async flush() {
+    // Flushing after the providers have been shut down throws "invalid attempt to force flush after shutdown".
+    if (this.stopPromise) {
+      return;
+    }
     await Promise.all([
       this.meterProvider.forceFlush(),
       this.loggerProvider?.forceFlush(),
@@ -120,7 +185,11 @@ export class OpenTelemetryClient implements TelemetryClient {
     ]);
   }
 
-  public async stop() {
+  public stop() {
+    return (this.stopPromise ??= this.doStop());
+  }
+
+  private async doStop() {
     this.nodejsMetricsMonitor?.stop();
 
     const flushAndShutdown = async (provider?: { forceFlush: () => Promise<void>; shutdown: () => Promise<void> }) => {
@@ -138,6 +207,19 @@ export class OpenTelemetryClient implements TelemetryClient {
     ]);
   }
 
+  public getTraceContext(): string | undefined {
+    const carrier: Record<string, string> = {};
+    propagation.inject(context.active(), carrier);
+    return carrier['traceparent'];
+  }
+
+  public extractPropagatedContext(traceContext: string): Context {
+    const extractedContext = propagation.extract(ROOT_CONTEXT, {
+      traceparent: traceContext,
+    });
+    return extractedContext;
+  }
+
   public static createMeterProvider(
     resource: IResource,
     exporters: Array<PeriodicExportingMetricReaderOptions>,
@@ -173,6 +255,23 @@ export class OpenTelemetryClient implements TelemetryClient {
             true,
           ),
         }),
+        // Pending-to-mined delay routinely exceeds the 1-minute ceiling of the generic `ms`
+        // view below under load, so it would saturate at 60s. Give this one metric wider
+        // buckets (1s to 10min). This must precede the generic `ms` view: when multiple views
+        // match an instrument, the SDK keeps the first-registered compatible storage, so the
+        // first view in this list wins the bucket boundaries.
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentName: 'aztec.mempool.tx_mined_delay',
+          instrumentUnit: 'ms',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [
+              1_000, 2_500, 5_000, 7_500, 10_000, 15_000, 30_000, 45_000, 60_000, 90_000, 120_000, 180_000, 300_000,
+              600_000,
+            ],
+            true,
+          ),
+        }),
         new View({
           instrumentType: InstrumentType.HISTOGRAM,
           instrumentUnit: 'ms',
@@ -266,6 +365,36 @@ export class OpenTelemetryClient implements TelemetryClient {
             true,
           ),
         }),
+        // L1 gas prices in gwei: priority fees ~0.01-10, base fees ~1-500, spikes to 1000+
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'gwei',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1_000],
+            true,
+          ),
+        }),
+        // L1 gas consumption: tx gas 100k-30M, calldata/blob gas varies
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'gas',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [
+              10_000, 50_000, 100_000, 250_000, 500_000, 1_000_000, 2_000_000, 5_000_000, 10_000_000, 15_000_000,
+              30_000_000,
+            ],
+            true,
+          ),
+        }),
+        // L1 tx total fee in ETH: typically 0.001 - 1 ETH
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'eth',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10],
+            true,
+          ),
+        }),
       ],
     });
   }
@@ -275,19 +404,40 @@ export class OpenTelemetryClient implements TelemetryClient {
       const tracerProvider = new NodeTracerProvider({
         resource,
         spanProcessors: config.tracesCollectorUrl
-          ? [new BatchSpanProcessor(new OTLPTraceExporter({ url: config.tracesCollectorUrl.href }))]
+          ? [
+              new MonitoredBatchSpanProcessor(new OTLPTraceExporter({ url: config.tracesCollectorUrl.href }), log, {
+                maxQueueSize: config.otelBspMaxQueueSize,
+                minTraceDurationMs: config.otelMinTraceDurationMs,
+              }),
+            ]
           : [],
       });
 
-      tracerProvider.register();
+      tracerProvider.register({
+        propagator: new W3CTraceContextPropagator(),
+      });
 
       const exporters: PeriodicExportingMetricReaderOptions[] = [];
       if (config.metricsCollectorUrl) {
+        // Default to a blacklist that is empty (allow all metrics)
+        let filter: string[] = [];
+        let mode: 'allow' | 'deny' = 'deny';
+        if (config.otelExcludeMetrics.length > 0) {
+          // Implement a blacklist as specified in config
+          log.info(`Excluding metrics from export: ${config.otelExcludeMetrics}`);
+          filter = config.otelExcludeMetrics;
+          mode = 'deny';
+        } else if (config.otelIncludeMetrics.length > 0) {
+          // Implement a whitelist as specified in config
+          log.info(`Including only specified metrics for export: ${config.otelIncludeMetrics}`);
+          filter = config.otelIncludeMetrics;
+          mode = 'allow';
+        }
         exporters.push({
           exporter: new OtelFilterMetricExporter(
             new OTLPMetricExporter({ url: config.metricsCollectorUrl.href }),
-            config.otelExcludeMetrics,
-            'deny',
+            filter,
+            mode,
           ),
           exportTimeoutMillis: config.otelExportTimeoutMs,
           exportIntervalMillis: config.otelCollectIntervalMs,

package/src/otel_propagation.ts

@@ -1,3 +1,5 @@
+import type { DiagnosticsMiddleware } from '@aztec/foundation/json-rpc/server';
+
 import { ROOT_CONTEXT, type Span, SpanKind, SpanStatusCode, propagation } from '@opentelemetry/api';
 import type Koa from 'koa';
 
@@ -17,7 +19,7 @@ export function getOtelJsonRpcPropagationMiddleware(
     const context = propagation.extract(ROOT_CONTEXT, ctx.request.headers);
     const method = (ctx.request.body as any)?.method;
     return tracer.startActiveSpan(
-      `JsonRpcServer.${method ?? 'unknown'}`,
+      `JsonRpcServer.${method ?? 'batch'}`,
       { kind: SpanKind.SERVER },
       context,
       async (span: Span): Promise<void> => {
@@ -48,3 +50,42 @@ export function getOtelJsonRpcPropagationMiddleware(
     );
   };
 }
+
+export function getOtelJsonRpcDiagnosticsMiddleware(): DiagnosticsMiddleware {
+  return function otelJsonRpcDiagnostics(ctx, next) {
+    const [namespace, method] = splitNamespace(ctx.method);
+    const scope = namespace ?? 'UnknownHandler';
+    const tracer = getTelemetryClient().getTracer(scope);
+    return tracer.startActiveSpan(
+      `${scope}.${method}`,
+      { kind: SpanKind.INTERNAL, attributes: { [ATTR_JSONRPC_METHOD]: ctx.method } },
+      async span => {
+        if (ctx.id !== null) {
+          span.setAttribute(ATTR_JSONRPC_REQUEST_ID, ctx.id);
+        }
+
+        try {
+          await next();
+          span.setStatus({ code: SpanStatusCode.OK });
+        } catch (err) {
+          span.setStatus({ code: SpanStatusCode.ERROR, message: err instanceof Error ? err.message : String(err) });
+          if (typeof err === 'string' || err instanceof Error) {
+            span.recordException(err);
+          }
+          throw err;
+        } finally {
+          span.end();
+        }
+      },
+    );
+  };
+}
+
+function splitNamespace(fullMethod: string): [namespace: string | undefined, method: string] {
+  const idx = fullMethod.indexOf('_');
+  if (idx > -1) {
+    return [fullMethod.slice(0, idx), fullMethod.slice(idx + 1)];
+  } else {
+    return [undefined, fullMethod];
+  }
+}

package/src/otel_resource.ts

@@ -7,10 +7,26 @@ import {
   osDetectorSync,
   serviceInstanceIdDetectorSync,
 } from '@opentelemetry/resources';
-import { SEMRESATTRS_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
+import { readFileSync } from 'fs';
+import { dirname, resolve } from 'path';
+import { fileURLToPath } from 'url';
 
 import { AZTEC_NODE_ROLE, AZTEC_REGISTRY_ADDRESS, AZTEC_ROLLUP_ADDRESS, AZTEC_ROLLUP_VERSION } from './attributes.js';
 
+/** Reads the Aztec client version from the release manifest. */
+function getAztecVersion(): string | undefined {
+  try {
+    const releasePleasePath = resolve(
+      dirname(fileURLToPath(import.meta.url)),
+      '../../../.release-please-manifest.json',
+    );
+    return JSON.parse(readFileSync(releasePleasePath, 'utf-8'))['.'];
+  } catch {
+    return undefined;
+  }
+}
+
 export function getOtelResource(): IResource {
   const resource = detectResourcesSync({
     detectors: [
@@ -42,7 +58,8 @@ const aztecNetworkDetectorSync: DetectorSync = {
     }
     const aztecAttributes = {
       // this gets overwritten by OTEL_RESOURCE_ATTRIBUTES (if set)
-      [SEMRESATTRS_SERVICE_NAME]: role ? `aztec-${role}` : undefined,
+      [ATTR_SERVICE_NAME]: role ? `aztec-${role}` : undefined,
+      [ATTR_SERVICE_VERSION]: getAztecVersion(),
       [AZTEC_NODE_ROLE]: role,
       [AZTEC_ROLLUP_VERSION]: process.env.ROLLUP_VERSION ?? 'canonical',
       [AZTEC_ROLLUP_ADDRESS]: process.env.ROLLUP_CONTRACT_ADDRESS,