@aztec-labs/telemetry-client 6.0.0-nightly.20260829

package/src/monitored_batch_span_processor.ts

@@ -0,0 +1,147 @@
+import type { Logger } from '@aztec-labs/foundation/log';
+import { type Context, SpanStatusCode, TraceFlags } from '@opentelemetry/api';
+import { type ExportResult, hrTimeToMilliseconds } from '@opentelemetry/core';
+import type { SpanExporter } from '@opentelemetry/sdk-trace-base';
+import { BatchSpanProcessor, type BufferConfig, type ReadableSpan, type Span } from '@opentelemetry/sdk-trace-node';
+
+/** Minimum interval between drop warnings to avoid log spam. */
+const DROP_WARNING_INTERVAL_MS = 30_000;
+
+const DEFAULT_MIN_TRACE_DURATION_MS = 10;
+
+const DEFAULT_MAX_QUEUE_SIZE = 65536;
+
+/** Cap on the per-export batch size, so a large queue can actually be drained instead of dribbling out
+ * at the SDK default of 512 spans per scheduled export. Kept <= maxQueueSize per the BatchSpanProcessor contract. */
+const DEFAULT_MAX_EXPORT_BATCH_SIZE = 4096;
+
+export type MonitoredBatchSpanProcessorConfig = BufferConfig & {
+  minTraceDurationMs?: number;
+};
+
+/** Shared counter of spans sitting in the SDK's queue. It is a plain object rather than a field on the
+ * processor because the exporter wrapper that decrements it is built before `super()` runs, when `this`
+ * is not yet available. */
+type QueueGauge = { size: number };
+
+/**
+ * Delegating exporter that reports how many spans left the queue.
+ *
+ * `BatchSpanProcessor` removes a batch from its internal buffer and hands it straight to
+ * `exporter.export()` — exactly once per batch — so this is an exact drain signal. Spans in flight are
+ * already off the queue, which matches how the SDK evaluates its own `maxQueueSize` limit.
+ */
+class DrainCountingExporter implements SpanExporter {
+  constructor(
+    private readonly inner: SpanExporter,
+    private readonly queue: QueueGauge,
+  ) {}
+
+  export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): void {
+    this.queue.size = Math.max(0, this.queue.size - spans.length);
+    this.inner.export(spans, resultCallback);
+  }
+
+  shutdown(): Promise<void> {
+    return this.inner.shutdown();
+  }
+
+  forceFlush(): Promise<void> {
+    return this.inner.forceFlush?.() ?? Promise.resolve();
+  }
+}
+
+/**
+ * Wraps BatchSpanProcessor to emit warnings when spans are dropped due to a full queue.
+ * The standard BatchSpanProcessor silently discards spans when its internal queue reaches
+ * maxQueueSize, making telemetry data loss invisible to operators.
+ */
+export class MonitoredBatchSpanProcessor extends BatchSpanProcessor {
+  private readonly maxQueueSize: number;
+  private readonly maxExportBatchSize: number;
+  private readonly minTraceDurationMs: number;
+  private readonly log: Logger;
+  private readonly queue: QueueGauge;
+
+  private droppedSinceLastWarning = 0;
+  private totalDropped = 0;
+  private lastWarningTime = 0;
+  private shuttingDown = false;
+
+  constructor(exporter: SpanExporter, log: Logger, config?: MonitoredBatchSpanProcessorConfig) {
+    const maxQueueSize = config?.maxQueueSize ?? DEFAULT_MAX_QUEUE_SIZE;
+    const maxExportBatchSize = Math.min(config?.maxExportBatchSize ?? DEFAULT_MAX_EXPORT_BATCH_SIZE, maxQueueSize);
+    const queue: QueueGauge = { size: 0 };
+    super(new DrainCountingExporter(exporter, queue), { ...config, maxQueueSize, maxExportBatchSize });
+    this.queue = queue;
+    this.maxQueueSize = maxQueueSize;
+    this.maxExportBatchSize = maxExportBatchSize;
+    this.minTraceDurationMs = Math.max(0, config?.minTraceDurationMs ?? DEFAULT_MIN_TRACE_DURATION_MS);
+    this.log = log;
+  }
+
+  override onStart(span: Span, parentContext: Context): void {
+    super.onStart(span, parentContext);
+  }
+
+  override onEnd(span: ReadableSpan): void {
+    if (this.shouldDropShortSpan(span)) {
+      return;
+    }
+
+    if (this.isQueued(span)) {
+      if (this.queue.size >= this.maxQueueSize) {
+        this.droppedSinceLastWarning++;
+        this.totalDropped++;
+        this.maybeLogDropWarning();
+      } else {
+        this.queue.size++;
+      }
+    }
+
+    super.onEnd(span);
+  }
+
+  override async shutdown(): Promise<void> {
+    this.shuttingDown = true;
+    if (this.totalDropped > 0) {
+      this.log.warn(`BatchSpanProcessor shutting down with ${this.totalDropped} total spans dropped`, {
+        totalDropped: this.totalDropped,
+      });
+    }
+    await super.shutdown();
+  }
+
+  private shouldDropShortSpan(span: ReadableSpan): boolean {
+    return (
+      this.minTraceDurationMs > 0 &&
+      span.status.code !== SpanStatusCode.ERROR &&
+      hrTimeToMilliseconds(span.duration) < this.minTraceDurationMs
+    );
+  }
+
+  /** Mirrors the guards the SDK applies before buffering a span. Spans it discards outright never reach the
+   * queue and never reach the exporter, so counting them would leak the estimate upwards forever. */
+  private isQueued(span: ReadableSpan): boolean {
+    return !this.shuttingDown && (span.spanContext().traceFlags & TraceFlags.SAMPLED) !== 0;
+  }
+
+  private maybeLogDropWarning(): void {
+    const now = Date.now();
+    if (now - this.lastWarningTime >= DROP_WARNING_INTERVAL_MS) {
+      this.log.warn(
+        `BatchSpanProcessor dropping spans: queue full (maxQueueSize=${this.maxQueueSize}, ` +
+          `maxExportBatchSize=${this.maxExportBatchSize}). ` +
+          `${this.droppedSinceLastWarning} dropped since last warning, ${this.totalDropped} total.`,
+        {
+          droppedSinceLastWarning: this.droppedSinceLastWarning,
+          totalDropped: this.totalDropped,
+          maxQueueSize: this.maxQueueSize,
+          maxExportBatchSize: this.maxExportBatchSize,
+        },
+      );
+      this.droppedSinceLastWarning = 0;
+      this.lastWarningTime = now;
+    }
+  }
+}

package/src/nodejs_metrics_monitor.ts

@@ -0,0 +1,135 @@
+import type { Observable } from '@opentelemetry/api';
+import { type EventLoopUtilization, type IntervalHistogram, monitorEventLoopDelay, performance } from 'node:perf_hooks';
+
+import * as Attributes from './attributes.js';
+import { createUpDownCounterWithDefault } from './metric-utils.js';
+import * as Metrics from './metrics.js';
+import type { BatchObservableResult, Meter, ObservableGauge, UpDownCounter } from './telemetry.js';
+
+/** Monitors Node.js runtime metrics */
+export class NodejsMetricsMonitor {
+  private eventLoopDelayGauges: {
+    min: ObservableGauge;
+    max: ObservableGauge;
+    mean: ObservableGauge;
+    stddev: ObservableGauge;
+    p50: ObservableGauge;
+    p90: ObservableGauge;
+    p99: ObservableGauge;
+  };
+
+  // skip `rss` because that's already tracked by @opentelemetry/host-metrics
+  // description of each field here https://nodejs.org/api/process.html#processmemoryusage
+  private memoryGauges: Record<Exclude<keyof NodeJS.MemoryUsage, 'rss'>, ObservableGauge>;
+
+  private eventLoopUilization: ObservableGauge;
+  private eventLoopTime: UpDownCounter;
+
+  private started = false;
+
+  private lastELU: EventLoopUtilization | undefined;
+  private eventLoopDelay: IntervalHistogram;
+
+  constructor(private meter: Meter) {
+    this.eventLoopDelayGauges = {
+      min: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MIN),
+      mean: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MEAN),
+      max: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MAX),
+      stddev: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_STDDEV),
+      p50: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P50),
+      p90: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P90),
+      p99: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P99),
+    };
+
+    this.eventLoopUilization = meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_UTILIZATION);
+    this.eventLoopTime = createUpDownCounterWithDefault(meter, Metrics.NODEJS_EVENT_LOOP_TIME, {
+      [Attributes.NODEJS_EVENT_LOOP_STATE]: ['idle', 'active'],
+    });
+    this.eventLoopDelay = monitorEventLoopDelay();
+
+    this.memoryGauges = {
+      heapUsed: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_USAGE),
+      heapTotal: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_TOTAL),
+      arrayBuffers: meter.createObservableGauge(Metrics.NODEJS_MEMORY_BUFFER_USAGE),
+      external: meter.createObservableGauge(Metrics.NODEJS_MEMORY_NATIVE_USAGE),
+    };
+  }
+
+  start(): void {
+    if (this.started) {
+      return;
+    }
+
+    this.lastELU = performance.eventLoopUtilization();
+    this.eventLoopDelay.enable();
+    this.meter.addBatchObservableCallback(this.measure, [
+      this.eventLoopUilization,
+      ...Object.values(this.eventLoopDelayGauges),
+      ...Object.values(this.memoryGauges),
+    ]);
+  }
+
+  stop(): void {
+    if (!this.started) {
+      return;
+    }
+    this.meter.removeBatchObservableCallback(this.measure, [
+      this.eventLoopUilization,
+      ...Object.values(this.eventLoopDelayGauges),
+      ...Object.values(this.memoryGauges),
+    ]);
+    this.eventLoopDelay.disable();
+    this.eventLoopDelay.reset();
+    this.lastELU = undefined;
+  }
+
+  private measure = (obs: BatchObservableResult): void => {
+    this.measureMemoryUsage(obs);
+    this.measureEventLoopDelay(obs);
+  };
+
+  private measureMemoryUsage = (observer: BatchObservableResult) => {
+    const mem = process.memoryUsage();
+
+    observer.observe(this.memoryGauges.heapUsed, mem.heapUsed);
+    observer.observe(this.memoryGauges.heapTotal, mem.heapTotal);
+    observer.observe(this.memoryGauges.arrayBuffers, mem.arrayBuffers);
+    observer.observe(this.memoryGauges.external, mem.external);
+  };
+
+  private measureEventLoopDelay = (obs: BatchObservableResult): void => {
+    const newELU = performance.eventLoopUtilization();
+    const delta = performance.eventLoopUtilization(newELU, this.lastELU);
+    this.lastELU = newELU;
+
+    // `utilization` [0,1] represents how much the event loop is busy vs waiting for new events to come in
+    // This should be corelated with CPU usage to gauge the performance characteristics of services
+    // 100% utilization leads to high latency because the event loop is _always_ busy, there's no breathing room for events to be processed quickly.
+    // Docs and examples:
+    // - https://nodesource.com/blog/event-loop-utilization-nodejs
+    // - https://youtu.be/WetXnEPraYM
+    obs.observe(this.eventLoopUilization, delta.utilization);
+
+    this.eventLoopTime.add(Math.trunc(delta.idle), { [Attributes.NODEJS_EVENT_LOOP_STATE]: 'idle' });
+    this.eventLoopTime.add(Math.trunc(delta.active), { [Attributes.NODEJS_EVENT_LOOP_STATE]: 'active' });
+
+    safeObserveInt(obs, this.eventLoopDelayGauges.min, this.eventLoopDelay.min);
+    safeObserveInt(obs, this.eventLoopDelayGauges.mean, this.eventLoopDelay.mean);
+    safeObserveInt(obs, this.eventLoopDelayGauges.max, this.eventLoopDelay.max);
+    safeObserveInt(obs, this.eventLoopDelayGauges.stddev, this.eventLoopDelay.stddev);
+    safeObserveInt(obs, this.eventLoopDelayGauges.p50, this.eventLoopDelay.percentile(50));
+    safeObserveInt(obs, this.eventLoopDelayGauges.p90, this.eventLoopDelay.percentile(90));
+    safeObserveInt(obs, this.eventLoopDelayGauges.p99, this.eventLoopDelay.percentile(99));
+
+    this.eventLoopDelay.reset();
+  };
+}
+
+function safeObserveInt(observer: BatchObservableResult, metric: Observable, value: number, attrs?: object) {
+  // discard NaN, Infinity, -Infinity
+  if (!Number.isFinite(value)) {
+    return;
+  }
+
+  observer.observe(metric, Math.trunc(value), attrs);
+}

package/src/noop.ts

@@ -0,0 +1,152 @@
+import { type Context, type Span, type SpanContext, type Tracer, createNoopMeter } from '@opentelemetry/api';
+
+import type { MetricDefinition } from './metrics.js';
+import type {
+  Gauge,
+  Histogram,
+  Meter,
+  ObservableGauge,
+  ObservableUpDownCounter,
+  TelemetryClient,
+  UpDownCounter,
+} from './telemetry.js';
+
+/** A no-op meter that implements our custom Meter interface */
+class NoopMeter implements Meter {
+  private otelMeter = createNoopMeter();
+
+  createGauge(_metric: MetricDefinition): Gauge {
+    return this.otelMeter.createGauge('');
+  }
+
+  createObservableGauge(_metric: MetricDefinition): ObservableGauge {
+    return this.otelMeter.createObservableGauge('');
+  }
+
+  createHistogram(_metric: MetricDefinition, _extraOptions?: Parameters<Meter['createHistogram']>[1]): Histogram {
+    return this.otelMeter.createHistogram('');
+  }
+
+  createUpDownCounter(_metric: MetricDefinition): UpDownCounter {
+    return this.otelMeter.createUpDownCounter('');
+  }
+
+  createObservableUpDownCounter(_metric: MetricDefinition): ObservableUpDownCounter {
+    return this.otelMeter.createObservableUpDownCounter('');
+  }
+
+  addBatchObservableCallback(
+    callback: Parameters<Meter['addBatchObservableCallback']>[0],
+    observables: Parameters<Meter['addBatchObservableCallback']>[1],
+  ): void {
+    this.otelMeter.addBatchObservableCallback(callback, observables);
+  }
+
+  removeBatchObservableCallback(
+    callback: Parameters<Meter['removeBatchObservableCallback']>[0],
+    observables: Parameters<Meter['removeBatchObservableCallback']>[1],
+  ): void {
+    this.otelMeter.removeBatchObservableCallback(callback, observables);
+  }
+}
+
+export class NoopTelemetryClient implements TelemetryClient {
+  private meter = new NoopMeter();
+
+  setExportedPublicTelemetry(_prefixes: string[]): void {}
+  setPublicTelemetryCollectFrom(_roles: string[]): void {}
+
+  getMeter(): Meter {
+    return this.meter;
+  }
+
+  getTracer(): Tracer {
+    return new NoopTracer();
+  }
+
+  stop(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  flush(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  isEnabled() {
+    return false;
+  }
+
+  getTraceContext(): string | undefined {
+    return undefined;
+  }
+
+  extractPropagatedContext(_traceContext: string): Context | undefined {
+    return undefined;
+  }
+}
+
+// @opentelemetry/api internally uses NoopTracer and NoopSpan but they're not exported
+// make our own versions
+// https://github.com/open-telemetry/opentelemetry-js/issues/4518#issuecomment-2179405444
+export class NoopTracer implements Tracer {
+  startSpan(): Span {
+    return new NoopSpan();
+  }
+
+  startActiveSpan<F extends (...args: any[]) => any>(_name: string, ...args: unknown[]): ReturnType<F> {
+    // there are three different signatures for startActiveSpan, grab the function, we don't care about the rest
+    const fn = args.find(arg => typeof arg === 'function') as F;
+    return fn(new NoopSpan());
+  }
+}
+
+class NoopSpan implements Span {
+  private recording: boolean = true;
+  addEvent(): this {
+    return this;
+  }
+
+  addLink(): this {
+    return this;
+  }
+
+  addLinks(): this {
+    return this;
+  }
+
+  end(): void {
+    this.recording = false;
+  }
+
+  isRecording(): boolean {
+    return this.recording;
+  }
+
+  recordException(): void {
+    return;
+  }
+
+  setAttribute(): this {
+    return this;
+  }
+
+  setAttributes(): this {
+    return this;
+  }
+
+  setStatus(): this {
+    return this;
+  }
+
+  spanContext(): SpanContext {
+    return {
+      spanId: '',
+      traceId: '',
+      traceFlags: 0,
+    };
+  }
+
+  updateName(): this {
+    return this;
+  }
+}