@aztec/telemetry-client 0.0.1-commit.96bb3f7 → 0.0.1-commit.993d240

package/src/monitored_batch_span_processor.ts

@@ -0,0 +1,93 @@
+import type { Logger } from '@aztec/foundation/log';
+
+import { type Context, SpanStatusCode } from '@opentelemetry/api';
+import { hrTimeToMilliseconds } from '@opentelemetry/core';
+import type { SpanExporter } from '@opentelemetry/sdk-trace-base';
+import { BatchSpanProcessor, type BufferConfig, type ReadableSpan, type Span } from '@opentelemetry/sdk-trace-node';
+
+/** Minimum interval between drop warnings to avoid log spam. */
+const DROP_WARNING_INTERVAL_MS = 30_000;
+
+const DEFAULT_MIN_TRACE_DURATION_MS = 10;
+
+export type MonitoredBatchSpanProcessorConfig = BufferConfig & {
+  minTraceDurationMs?: number;
+};
+
+/**
+ * Wraps BatchSpanProcessor to emit warnings when spans are dropped due to a full queue.
+ * The standard BatchSpanProcessor silently discards spans when its internal queue reaches
+ * maxQueueSize, making telemetry data loss invisible to operators.
+ */
+export class MonitoredBatchSpanProcessor extends BatchSpanProcessor {
+  private readonly maxQueueSize: number;
+  private readonly minTraceDurationMs: number;
+  private readonly log: Logger;
+
+  private approxQueueSize = 0;
+  private droppedSinceLastWarning = 0;
+  private totalDropped = 0;
+  private lastWarningTime = 0;
+
+  constructor(exporter: SpanExporter, log: Logger, config?: MonitoredBatchSpanProcessorConfig) {
+    const maxQueueSize = config?.maxQueueSize ?? 2048;
+    super(exporter, { ...config, maxQueueSize });
+    this.maxQueueSize = maxQueueSize;
+    this.minTraceDurationMs = Math.max(0, config?.minTraceDurationMs ?? DEFAULT_MIN_TRACE_DURATION_MS);
+    this.log = log;
+  }
+
+  override onStart(span: Span, parentContext: Context): void {
+    super.onStart(span, parentContext);
+  }
+
+  override onEnd(span: ReadableSpan): void {
+    if (this.shouldDropShortSpan(span)) {
+      return;
+    }
+
+    if (this.approxQueueSize >= this.maxQueueSize) {
+      this.droppedSinceLastWarning++;
+      this.totalDropped++;
+      this.maybeLogDropWarning();
+    } else {
+      this.approxQueueSize++;
+    }
+    super.onEnd(span);
+  }
+
+  override async forceFlush(): Promise<void> {
+    await super.forceFlush();
+    this.approxQueueSize = 0;
+  }
+
+  override async shutdown(): Promise<void> {
+    if (this.totalDropped > 0) {
+      this.log.warn(`BatchSpanProcessor shutting down with ${this.totalDropped} total spans dropped`, {
+        totalDropped: this.totalDropped,
+      });
+    }
+    await super.shutdown();
+  }
+
+  private shouldDropShortSpan(span: ReadableSpan): boolean {
+    return (
+      this.minTraceDurationMs > 0 &&
+      span.status.code !== SpanStatusCode.ERROR &&
+      hrTimeToMilliseconds(span.duration) < this.minTraceDurationMs
+    );
+  }
+
+  private maybeLogDropWarning(): void {
+    const now = Date.now();
+    if (now - this.lastWarningTime >= DROP_WARNING_INTERVAL_MS) {
+      this.log.warn(
+        `BatchSpanProcessor dropping spans: queue full (maxQueueSize=${this.maxQueueSize}). ` +
+          `${this.droppedSinceLastWarning} dropped since last warning, ${this.totalDropped} total.`,
+        { droppedSinceLastWarning: this.droppedSinceLastWarning, totalDropped: this.totalDropped },
+      );
+      this.droppedSinceLastWarning = 0;
+      this.lastWarningTime = now;
+    }
+  }
+}

package/src/nodejs_metrics_monitor.ts

@@ -2,6 +2,7 @@ import type { Observable } from '@opentelemetry/api';
 import { type EventLoopUtilization, type IntervalHistogram, monitorEventLoopDelay, performance } from 'node:perf_hooks';
 
 import * as Attributes from './attributes.js';
+import { createUpDownCounterWithDefault } from './metric-utils.js';
 import * as Metrics from './metrics.js';
 import type { BatchObservableResult, Meter, ObservableGauge, UpDownCounter } from './telemetry.js';
 
@@ -41,7 +42,9 @@ export class NodejsMetricsMonitor {
     };
 
     this.eventLoopUilization = meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_UTILIZATION);
-    this.eventLoopTime = meter.createUpDownCounter(Metrics.NODEJS_EVENT_LOOP_TIME);
+    this.eventLoopTime = createUpDownCounterWithDefault(meter, Metrics.NODEJS_EVENT_LOOP_TIME, {
+      [Attributes.NODEJS_EVENT_LOOP_STATE]: ['idle', 'active'],
+    });
     this.eventLoopDelay = monitorEventLoopDelay();
 
     this.memoryGauges = {

package/src/otel.ts

@@ -28,12 +28,13 @@ import {
   type PeriodicExportingMetricReaderOptions,
   View,
 } from '@opentelemetry/sdk-metrics';
-import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
 import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
 
 import type { TelemetryClientConfig } from './config.js';
 import { toMetricOptions } from './metric-utils.js';
 import type { MetricDefinition } from './metrics.js';
+import { MonitoredBatchSpanProcessor } from './monitored_batch_span_processor.js';
 import { NodejsMetricsMonitor } from './nodejs_metrics_monitor.js';
 import { OtelFilterMetricExporter, PublicOtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
 import { registerOtelLoggerProvider } from './otel_logger_provider.js';
@@ -334,6 +335,36 @@ export class OpenTelemetryClient implements TelemetryClient {
             true,
           ),
         }),
+        // L1 gas prices in gwei: priority fees ~0.01-10, base fees ~1-500, spikes to 1000+
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'gwei',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1_000],
+            true,
+          ),
+        }),
+        // L1 gas consumption: tx gas 100k-30M, calldata/blob gas varies
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'gas',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [
+              10_000, 50_000, 100_000, 250_000, 500_000, 1_000_000, 2_000_000, 5_000_000, 10_000_000, 15_000_000,
+              30_000_000,
+            ],
+            true,
+          ),
+        }),
+        // L1 tx total fee in ETH: typically 0.001 - 1 ETH
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'eth',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10],
+            true,
+          ),
+        }),
       ],
     });
   }
@@ -343,7 +374,12 @@ export class OpenTelemetryClient implements TelemetryClient {
       const tracerProvider = new NodeTracerProvider({
         resource,
         spanProcessors: config.tracesCollectorUrl
-          ? [new BatchSpanProcessor(new OTLPTraceExporter({ url: config.tracesCollectorUrl.href }))]
+          ? [
+              new MonitoredBatchSpanProcessor(new OTLPTraceExporter({ url: config.tracesCollectorUrl.href }), log, {
+                maxQueueSize: config.otelBspMaxQueueSize,
+                minTraceDurationMs: config.otelMinTraceDurationMs,
+              }),
+            ]
           : [],
       });
 

package/src/otel_propagation.ts

@@ -1,3 +1,5 @@
+import type { DiagnosticsMiddleware } from '@aztec/foundation/json-rpc/server';
+
 import { ROOT_CONTEXT, type Span, SpanKind, SpanStatusCode, propagation } from '@opentelemetry/api';
 import type Koa from 'koa';
 
@@ -17,7 +19,7 @@ export function getOtelJsonRpcPropagationMiddleware(
     const context = propagation.extract(ROOT_CONTEXT, ctx.request.headers);
     const method = (ctx.request.body as any)?.method;
     return tracer.startActiveSpan(
-      `JsonRpcServer.${method ?? 'unknown'}`,
+      `JsonRpcServer.${method ?? 'batch'}`,
       { kind: SpanKind.SERVER },
       context,
       async (span: Span): Promise<void> => {
@@ -48,3 +50,42 @@ export function getOtelJsonRpcPropagationMiddleware(
     );
   };
 }
+
+export function getOtelJsonRpcDiagnosticsMiddleware(): DiagnosticsMiddleware {
+  return function otelJsonRpcDiagnostics(ctx, next) {
+    const [namespace, method] = splitNamespace(ctx.method);
+    const scope = namespace ?? 'UnknownHandler';
+    const tracer = getTelemetryClient().getTracer(scope);
+    return tracer.startActiveSpan(
+      `${scope}.${method}`,
+      { kind: SpanKind.INTERNAL, attributes: { [ATTR_JSONRPC_METHOD]: ctx.method } },
+      async span => {
+        if (ctx.id !== null) {
+          span.setAttribute(ATTR_JSONRPC_REQUEST_ID, ctx.id);
+        }
+
+        try {
+          await next();
+          span.setStatus({ code: SpanStatusCode.OK });
+        } catch (err) {
+          span.setStatus({ code: SpanStatusCode.ERROR, message: err instanceof Error ? err.message : String(err) });
+          if (typeof err === 'string' || err instanceof Error) {
+            span.recordException(err);
+          }
+          throw err;
+        } finally {
+          span.end();
+        }
+      },
+    );
+  };
+}
+
+function splitNamespace(fullMethod: string): [namespace: string | undefined, method: string] {
+  const idx = fullMethod.indexOf('_');
+  if (idx > -1) {
+    return [fullMethod.slice(0, idx), fullMethod.slice(idx + 1)];
+  } else {
+    return [undefined, fullMethod];
+  }
+}

package/src/otel_resource.ts

@@ -7,10 +7,26 @@ import {
   osDetectorSync,
   serviceInstanceIdDetectorSync,
 } from '@opentelemetry/resources';
-import { SEMRESATTRS_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
+import { readFileSync } from 'fs';
+import { dirname, resolve } from 'path';
+import { fileURLToPath } from 'url';
 
 import { AZTEC_NODE_ROLE, AZTEC_REGISTRY_ADDRESS, AZTEC_ROLLUP_ADDRESS, AZTEC_ROLLUP_VERSION } from './attributes.js';
 
+/** Reads the Aztec client version from the release manifest. */
+function getAztecVersion(): string | undefined {
+  try {
+    const releasePleasePath = resolve(
+      dirname(fileURLToPath(import.meta.url)),
+      '../../../.release-please-manifest.json',
+    );
+    return JSON.parse(readFileSync(releasePleasePath, 'utf-8'))['.'];
+  } catch {
+    return undefined;
+  }
+}
+
 export function getOtelResource(): IResource {
   const resource = detectResourcesSync({
     detectors: [
@@ -42,7 +58,8 @@ const aztecNetworkDetectorSync: DetectorSync = {
     }
     const aztecAttributes = {
       // this gets overwritten by OTEL_RESOURCE_ATTRIBUTES (if set)
-      [SEMRESATTRS_SERVICE_NAME]: role ? `aztec-${role}` : undefined,
+      [ATTR_SERVICE_NAME]: role ? `aztec-${role}` : undefined,
+      [ATTR_SERVICE_VERSION]: getAztecVersion(),
       [AZTEC_NODE_ROLE]: role,
       [AZTEC_ROLLUP_VERSION]: process.env.ROLLUP_VERSION ?? 'canonical',
       [AZTEC_ROLLUP_ADDRESS]: process.env.ROLLUP_CONTRACT_ADDRESS,

package/src/prom_otel_adapter.ts

@@ -1,4 +1,4 @@
-import { type Logger, createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { Timer } from '@aztec/foundation/timer';
 
 import { Registry } from 'prom-client';
@@ -410,12 +410,11 @@ function parseLabelsSafely<Labels extends LabelsGeneric>(labelStr: string, logge
  */
 export class OtelMetricsAdapter extends Registry implements MetricsRegister {
   private readonly meter: Meter;
+  private logger: Logger;
 
-  constructor(
-    telemetryClient: TelemetryClient,
-    private logger: Logger = createLogger('telemetry:otel-metrics-adapter'),
-  ) {
+  constructor(telemetryClient: TelemetryClient, bindings?: LoggerBindings) {
     super();
+    this.logger = createLogger('telemetry:otel-metrics-adapter', bindings);
     this.meter = telemetryClient.getMeter('metrics-adapter');
   }
 

package/src/start.ts

@@ -1,4 +1,4 @@
-import { createLogger } from '@aztec/foundation/log';
+import { type LoggerBindings, createLogger } from '@aztec/foundation/log';
 
 import type { TelemetryClientConfig } from './config.js';
 import { NoopTelemetryClient } from './noop.js';
@@ -9,14 +9,22 @@ export * from './config.js';
 let initialized = false;
 let telemetry: TelemetryClient = new NoopTelemetryClient();
 
-export async function initTelemetryClient(config: TelemetryClientConfig): Promise<TelemetryClient> {
-  const log = createLogger('telemetry:client');
+export async function initTelemetryClient(
+  config: TelemetryClientConfig,
+  bindings?: LoggerBindings,
+): Promise<TelemetryClient> {
+  const log = createLogger('telemetry:client', bindings);
   if (initialized) {
     log.warn('Telemetry client has already been initialized once');
     return telemetry;
   }
 
-  if (config.metricsCollectorUrl || config.publicMetricsCollectorUrl) {
+  if (
+    config.metricsCollectorUrl ||
+    config.publicMetricsCollectorUrl ||
+    config.tracesCollectorUrl ||
+    config.logsCollectorUrl
+  ) {
     log.info(`Using OpenTelemetry client with custom collector`);
     // Lazy load OpenTelemetry to avoid loading heavy deps at startup
     const { OpenTelemetryClient } = await import('./otel.js');

package/src/telemetry.ts

@@ -20,7 +20,7 @@ import type * as Attributes from './attributes.js';
 import type { MetricDefinition } from './metrics.js';
 import { getTelemetryClient } from './start.js';
 
-export { toMetricOptions } from './metric-utils.js';
+export { toMetricOptions, createUpDownCounterWithDefault } from './metric-utils.js';
 
 export { type Span, SpanStatusCode, ValueType, type Context } from '@opentelemetry/api';
 
@@ -48,7 +48,6 @@ type BannedMetricAttributeNames = (typeof Attributes)[
   | 'TX_HASH'
   | 'PROVING_JOB_ID'
   | 'P2P_ID'
-  | 'P2P_REQ_RESP_BATCH_REQUESTS_COUNT'
   | 'TARGET_ADDRESS'
   | 'MANA_USED'
   | 'TOTAL_INSTRUCTIONS'];
@@ -56,8 +55,10 @@ type BannedMetricAttributeNames = (typeof Attributes)[
 /** Global registry of attributes */
 export type AttributesType = Partial<Record<AttributeNames, AttributeValue>>;
 
+export type AllowedAttributeNames = Exclude<AttributeNames, BannedMetricAttributeNames>;
+
 /** Subset of attributes allowed to be added to metrics */
-export type MetricAttributesType = Partial<Record<Exclude<AttributeNames, BannedMetricAttributeNames>, AttributeValue>>;
+export type MetricAttributesType = Partial<Record<AllowedAttributeNames, AttributeValue>>;
 
 /** Re-export MetricDefinition for convenience */
 export type { MetricDefinition } from './metrics.js';

package/src/wrappers/fetch.ts

@@ -9,12 +9,17 @@ import { ATTR_JSONRPC_METHOD, ATTR_JSONRPC_REQUEST_ID } from '../vendor/attribut
 
 /**
  * Makes a fetch function that retries based on the given attempts and propagates trace information.
- * @param retries - Sequence of intervals (in seconds) to retry.
+ * @param retries - Sequence of intervals (in seconds) to retry, or a factory function returning an iterator for custom/indefinite backoff.
  * @param noRetry - Whether to stop retries on server errors.
  * @param log - Optional logger for logging attempts.
  * @returns A fetch function.
  */
-export function makeTracedFetch(retries: number[], defaultNoRetry: boolean, fetch = defaultFetch, log?: Logger) {
+export function makeTracedFetch(
+  retries: number[] | (() => Generator<number>),
+  defaultNoRetry: boolean,
+  fetch = defaultFetch,
+  log?: Logger,
+) {
   return (host: string, body: unknown, extraHeaders: Record<string, string> = {}, noRetry?: boolean) => {
     const telemetry = getTelemetryClient();
     return telemetry.getTracer('fetch').startActiveSpan(`JsonRpcClient`, { kind: SpanKind.CLIENT }, async span => {
@@ -27,10 +32,11 @@ export function makeTracedFetch(retries: number[], defaultNoRetry: boolean, fetc
         }
         const headers = { ...extraHeaders };
         propagation.inject(context.active(), headers);
+        const backoff = typeof retries === 'function' ? retries() : makeBackoff(retries);
         return await retry(
           () => fetch(host, body, headers, noRetry ?? defaultNoRetry),
           `JsonRpcClient request to ${host}`,
-          makeBackoff(retries),
+          backoff,
           log,
           false,
         );

package/src/wrappers/l2_block_stream.ts

@@ -11,10 +11,7 @@ import { type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client'
 /** Extends an L2BlockStream with a tracer to create a new trace per iteration. */
 export class TraceableL2BlockStream extends L2BlockStream implements Traceable {
   constructor(
-    l2BlockSource: Pick<
-      L2BlockSource,
-      'getL2BlocksNew' | 'getBlockHeader' | 'getL2Tips' | 'getPublishedCheckpoints' | 'getCheckpointedBlocks'
-    >,
+    l2BlockSource: Pick<L2BlockSource, 'getBlocks' | 'getBlockData' | 'getL2Tips' | 'getCheckpoints'>,
     localData: L2BlockStreamLocalDataProvider,
     handler: L2BlockStreamEventHandler,
     public readonly tracer: Tracer,