@aztec/telemetry-client 0.0.1-commit.179ed20 → 0.0.1-commit.189eedb3

package/src/otel.ts

@@ -28,12 +28,13 @@ import {
   type PeriodicExportingMetricReaderOptions,
   View,
 } from '@opentelemetry/sdk-metrics';
-import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
 import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
 
 import type { TelemetryClientConfig } from './config.js';
 import { toMetricOptions } from './metric-utils.js';
 import type { MetricDefinition } from './metrics.js';
+import { MonitoredBatchSpanProcessor } from './monitored_batch_span_processor.js';
 import { NodejsMetricsMonitor } from './nodejs_metrics_monitor.js';
 import { OtelFilterMetricExporter, PublicOtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
 import { registerOtelLoggerProvider } from './otel_logger_provider.js';
@@ -95,6 +96,11 @@ export class OpenTelemetryClient implements TelemetryClient {
   private meters: Map<string, WrappedMeter> = new Map<string, WrappedMeter>();
   private tracers: Map<string, Tracer> = new Map<string, Tracer>();
 
+  /** Memoized shutdown promise. The telemetry client is shared between the aztec-node and an embedded prover-node,
+   * so stop() can be invoked more than once; the providers throw "shutdown may only be called once" and
+   * "invalid attempt to force flush after shutdown" if that happens. Guarding here makes stop()/flush() idempotent. */
+  private stopPromise: Promise<void> | undefined;
+
   protected constructor(
     private resource: IResource,
     private meterProvider: MeterProvider,
@@ -168,6 +174,10 @@ export class OpenTelemetryClient implements TelemetryClient {
   }
 
   public async flush() {
+    // Flushing after the providers have been shut down throws "invalid attempt to force flush after shutdown".
+    if (this.stopPromise) {
+      return;
+    }
     await Promise.all([
       this.meterProvider.forceFlush(),
       this.loggerProvider?.forceFlush(),
@@ -175,7 +185,11 @@ export class OpenTelemetryClient implements TelemetryClient {
     ]);
   }
 
-  public async stop() {
+  public stop() {
+    return (this.stopPromise ??= this.doStop());
+  }
+
+  private async doStop() {
     this.nodejsMetricsMonitor?.stop();
 
     const flushAndShutdown = async (provider?: { forceFlush: () => Promise<void>; shutdown: () => Promise<void> }) => {
@@ -334,6 +348,36 @@ export class OpenTelemetryClient implements TelemetryClient {
             true,
           ),
         }),
+        // L1 gas prices in gwei: priority fees ~0.01-10, base fees ~1-500, spikes to 1000+
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'gwei',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1_000],
+            true,
+          ),
+        }),
+        // L1 gas consumption: tx gas 100k-30M, calldata/blob gas varies
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'gas',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [
+              10_000, 50_000, 100_000, 250_000, 500_000, 1_000_000, 2_000_000, 5_000_000, 10_000_000, 15_000_000,
+              30_000_000,
+            ],
+            true,
+          ),
+        }),
+        // L1 tx total fee in ETH: typically 0.001 - 1 ETH
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'eth',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10],
+            true,
+          ),
+        }),
       ],
     });
   }
@@ -343,7 +387,12 @@ export class OpenTelemetryClient implements TelemetryClient {
       const tracerProvider = new NodeTracerProvider({
         resource,
         spanProcessors: config.tracesCollectorUrl
-          ? [new BatchSpanProcessor(new OTLPTraceExporter({ url: config.tracesCollectorUrl.href }))]
+          ? [
+              new MonitoredBatchSpanProcessor(new OTLPTraceExporter({ url: config.tracesCollectorUrl.href }), log, {
+                maxQueueSize: config.otelBspMaxQueueSize,
+                minTraceDurationMs: config.otelMinTraceDurationMs,
+              }),
+            ]
           : [],
       });
 

package/src/otel_propagation.ts

@@ -1,3 +1,5 @@
+import type { DiagnosticsMiddleware } from '@aztec/foundation/json-rpc/server';
+
 import { ROOT_CONTEXT, type Span, SpanKind, SpanStatusCode, propagation } from '@opentelemetry/api';
 import type Koa from 'koa';
 
@@ -17,7 +19,7 @@ export function getOtelJsonRpcPropagationMiddleware(
     const context = propagation.extract(ROOT_CONTEXT, ctx.request.headers);
     const method = (ctx.request.body as any)?.method;
     return tracer.startActiveSpan(
-      `JsonRpcServer.${method ?? 'unknown'}`,
+      `JsonRpcServer.${method ?? 'batch'}`,
       { kind: SpanKind.SERVER },
       context,
       async (span: Span): Promise<void> => {
@@ -48,3 +50,42 @@ export function getOtelJsonRpcPropagationMiddleware(
     );
   };
 }
+
+export function getOtelJsonRpcDiagnosticsMiddleware(): DiagnosticsMiddleware {
+  return function otelJsonRpcDiagnostics(ctx, next) {
+    const [namespace, method] = splitNamespace(ctx.method);
+    const scope = namespace ?? 'UnknownHandler';
+    const tracer = getTelemetryClient().getTracer(scope);
+    return tracer.startActiveSpan(
+      `${scope}.${method}`,
+      { kind: SpanKind.INTERNAL, attributes: { [ATTR_JSONRPC_METHOD]: ctx.method } },
+      async span => {
+        if (ctx.id !== null) {
+          span.setAttribute(ATTR_JSONRPC_REQUEST_ID, ctx.id);
+        }
+
+        try {
+          await next();
+          span.setStatus({ code: SpanStatusCode.OK });
+        } catch (err) {
+          span.setStatus({ code: SpanStatusCode.ERROR, message: err instanceof Error ? err.message : String(err) });
+          if (typeof err === 'string' || err instanceof Error) {
+            span.recordException(err);
+          }
+          throw err;
+        } finally {
+          span.end();
+        }
+      },
+    );
+  };
+}
+
+function splitNamespace(fullMethod: string): [namespace: string | undefined, method: string] {
+  const idx = fullMethod.indexOf('_');
+  if (idx > -1) {
+    return [fullMethod.slice(0, idx), fullMethod.slice(idx + 1)];
+  } else {
+    return [undefined, fullMethod];
+  }
+}

package/src/start.ts

@@ -19,7 +19,12 @@ export async function initTelemetryClient(
     return telemetry;
   }
 
-  if (config.metricsCollectorUrl || config.publicMetricsCollectorUrl) {
+  if (
+    config.metricsCollectorUrl ||
+    config.publicMetricsCollectorUrl ||
+    config.tracesCollectorUrl ||
+    config.logsCollectorUrl
+  ) {
     log.info(`Using OpenTelemetry client with custom collector`);
     // Lazy load OpenTelemetry to avoid loading heavy deps at startup
     const { OpenTelemetryClient } = await import('./otel.js');

package/src/telemetry.ts

@@ -48,7 +48,6 @@ type BannedMetricAttributeNames = (typeof Attributes)[
   | 'TX_HASH'
   | 'PROVING_JOB_ID'
   | 'P2P_ID'
-  | 'P2P_REQ_RESP_BATCH_REQUESTS_COUNT'
   | 'TARGET_ADDRESS'
   | 'MANA_USED'
   | 'TOTAL_INSTRUCTIONS'];

package/src/wrappers/fetch.ts

@@ -9,12 +9,17 @@ import { ATTR_JSONRPC_METHOD, ATTR_JSONRPC_REQUEST_ID } from '../vendor/attribut
 
 /**
  * Makes a fetch function that retries based on the given attempts and propagates trace information.
- * @param retries - Sequence of intervals (in seconds) to retry.
+ * @param retries - Sequence of intervals (in seconds) to retry, or a factory function returning an iterator for custom/indefinite backoff.
  * @param noRetry - Whether to stop retries on server errors.
  * @param log - Optional logger for logging attempts.
  * @returns A fetch function.
  */
-export function makeTracedFetch(retries: number[], defaultNoRetry: boolean, fetch = defaultFetch, log?: Logger) {
+export function makeTracedFetch(
+  retries: number[] | (() => Generator<number>),
+  defaultNoRetry: boolean,
+  fetch = defaultFetch,
+  log?: Logger,
+) {
   return (host: string, body: unknown, extraHeaders: Record<string, string> = {}, noRetry?: boolean) => {
     const telemetry = getTelemetryClient();
     return telemetry.getTracer('fetch').startActiveSpan(`JsonRpcClient`, { kind: SpanKind.CLIENT }, async span => {
@@ -27,10 +32,11 @@ export function makeTracedFetch(retries: number[], defaultNoRetry: boolean, fetc
         }
         const headers = { ...extraHeaders };
         propagation.inject(context.active(), headers);
+        const backoff = typeof retries === 'function' ? retries() : makeBackoff(retries);
         return await retry(
           () => fetch(host, body, headers, noRetry ?? defaultNoRetry),
           `JsonRpcClient request to ${host}`,
-          makeBackoff(retries),
+          backoff,
           log,
           false,
         );

package/src/wrappers/l2_block_stream.ts

@@ -11,10 +11,7 @@ import { type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client'
 /** Extends an L2BlockStream with a tracer to create a new trace per iteration. */
 export class TraceableL2BlockStream extends L2BlockStream implements Traceable {
   constructor(
-    l2BlockSource: Pick<
-      L2BlockSource,
-      'getBlocks' | 'getBlockHeader' | 'getL2Tips' | 'getCheckpoints' | 'getCheckpointedBlocks'
-    >,
+    l2BlockSource: Pick<L2BlockSource, 'getBlocks' | 'getBlockData' | 'getL2Tips' | 'getCheckpoints'>,
     localData: L2BlockStreamLocalDataProvider,
     handler: L2BlockStreamEventHandler,
     public readonly tracer: Tracer,