@aztec/telemetry-client 0.0.1-commit.2ed92850 → 0.0.1-commit.3100065

package/src/nodejs_metrics_monitor.ts

@@ -2,6 +2,7 @@ import type { Observable } from '@opentelemetry/api';
 import { type EventLoopUtilization, type IntervalHistogram, monitorEventLoopDelay, performance } from 'node:perf_hooks';
 
 import * as Attributes from './attributes.js';
+import { createUpDownCounterWithDefault } from './metric-utils.js';
 import * as Metrics from './metrics.js';
 import type { BatchObservableResult, Meter, ObservableGauge, UpDownCounter } from './telemetry.js';
 
@@ -41,7 +42,9 @@ export class NodejsMetricsMonitor {
     };
 
     this.eventLoopUilization = meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_UTILIZATION);
-    this.eventLoopTime = meter.createUpDownCounter(Metrics.NODEJS_EVENT_LOOP_TIME);
+    this.eventLoopTime = createUpDownCounterWithDefault(meter, Metrics.NODEJS_EVENT_LOOP_TIME, {
+      [Attributes.NODEJS_EVENT_LOOP_STATE]: ['idle', 'active'],
+    });
     this.eventLoopDelay = monitorEventLoopDelay();
 
     this.memoryGauges = {

package/src/otel.ts

@@ -28,12 +28,13 @@ import {
   type PeriodicExportingMetricReaderOptions,
   View,
 } from '@opentelemetry/sdk-metrics';
-import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
 import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
 
 import type { TelemetryClientConfig } from './config.js';
 import { toMetricOptions } from './metric-utils.js';
 import type { MetricDefinition } from './metrics.js';
+import { MonitoredBatchSpanProcessor } from './monitored_batch_span_processor.js';
 import { NodejsMetricsMonitor } from './nodejs_metrics_monitor.js';
 import { OtelFilterMetricExporter, PublicOtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
 import { registerOtelLoggerProvider } from './otel_logger_provider.js';
@@ -95,6 +96,11 @@ export class OpenTelemetryClient implements TelemetryClient {
   private meters: Map<string, WrappedMeter> = new Map<string, WrappedMeter>();
   private tracers: Map<string, Tracer> = new Map<string, Tracer>();
 
+  /** Memoized shutdown promise. The telemetry client is shared between the aztec-node and an embedded prover-node,
+   * so stop() can be invoked more than once; the providers throw "shutdown may only be called once" and
+   * "invalid attempt to force flush after shutdown" if that happens. Guarding here makes stop()/flush() idempotent. */
+  private stopPromise: Promise<void> | undefined;
+
   protected constructor(
     private resource: IResource,
     private meterProvider: MeterProvider,
@@ -168,6 +174,10 @@ export class OpenTelemetryClient implements TelemetryClient {
   }
 
   public async flush() {
+    // Flushing after the providers have been shut down throws "invalid attempt to force flush after shutdown".
+    if (this.stopPromise) {
+      return;
+    }
     await Promise.all([
       this.meterProvider.forceFlush(),
       this.loggerProvider?.forceFlush(),
@@ -175,7 +185,11 @@ export class OpenTelemetryClient implements TelemetryClient {
     ]);
   }
 
-  public async stop() {
+  public stop() {
+    return (this.stopPromise ??= this.doStop());
+  }
+
+  private async doStop() {
     this.nodejsMetricsMonitor?.stop();
 
     const flushAndShutdown = async (provider?: { forceFlush: () => Promise<void>; shutdown: () => Promise<void> }) => {
@@ -241,6 +255,23 @@ export class OpenTelemetryClient implements TelemetryClient {
             true,
           ),
         }),
+        // Pending-to-mined delay routinely exceeds the 1-minute ceiling of the generic `ms`
+        // view below under load, so it would saturate at 60s. Give this one metric wider
+        // buckets (1s to 10min). This must precede the generic `ms` view: when multiple views
+        // match an instrument, the SDK keeps the first-registered compatible storage, so the
+        // first view in this list wins the bucket boundaries.
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentName: 'aztec.mempool.tx_mined_delay',
+          instrumentUnit: 'ms',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [
+              1_000, 2_500, 5_000, 7_500, 10_000, 15_000, 30_000, 45_000, 60_000, 90_000, 120_000, 180_000, 300_000,
+              600_000,
+            ],
+            true,
+          ),
+        }),
         new View({
           instrumentType: InstrumentType.HISTOGRAM,
           instrumentUnit: 'ms',
@@ -334,6 +365,36 @@ export class OpenTelemetryClient implements TelemetryClient {
             true,
           ),
         }),
+        // L1 gas prices in gwei: priority fees ~0.01-10, base fees ~1-500, spikes to 1000+
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'gwei',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1_000],
+            true,
+          ),
+        }),
+        // L1 gas consumption: tx gas 100k-30M, calldata/blob gas varies
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'gas',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [
+              10_000, 50_000, 100_000, 250_000, 500_000, 1_000_000, 2_000_000, 5_000_000, 10_000_000, 15_000_000,
+              30_000_000,
+            ],
+            true,
+          ),
+        }),
+        // L1 tx total fee in ETH: typically 0.001 - 1 ETH
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'eth',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10],
+            true,
+          ),
+        }),
       ],
     });
   }
@@ -343,7 +404,12 @@ export class OpenTelemetryClient implements TelemetryClient {
       const tracerProvider = new NodeTracerProvider({
         resource,
         spanProcessors: config.tracesCollectorUrl
-          ? [new BatchSpanProcessor(new OTLPTraceExporter({ url: config.tracesCollectorUrl.href }))]
+          ? [
+              new MonitoredBatchSpanProcessor(new OTLPTraceExporter({ url: config.tracesCollectorUrl.href }), log, {
+                maxQueueSize: config.otelBspMaxQueueSize,
+                minTraceDurationMs: config.otelMinTraceDurationMs,
+              }),
+            ]
           : [],
       });
 

package/src/otel_propagation.ts

@@ -1,3 +1,5 @@
+import type { DiagnosticsMiddleware } from '@aztec/foundation/json-rpc/server';
+
 import { ROOT_CONTEXT, type Span, SpanKind, SpanStatusCode, propagation } from '@opentelemetry/api';
 import type Koa from 'koa';
 
@@ -17,7 +19,7 @@ export function getOtelJsonRpcPropagationMiddleware(
     const context = propagation.extract(ROOT_CONTEXT, ctx.request.headers);
     const method = (ctx.request.body as any)?.method;
     return tracer.startActiveSpan(
-      `JsonRpcServer.${method ?? 'unknown'}`,
+      `JsonRpcServer.${method ?? 'batch'}`,
       { kind: SpanKind.SERVER },
       context,
       async (span: Span): Promise<void> => {
@@ -48,3 +50,42 @@ export function getOtelJsonRpcPropagationMiddleware(
     );
   };
 }
+
+export function getOtelJsonRpcDiagnosticsMiddleware(): DiagnosticsMiddleware {
+  return function otelJsonRpcDiagnostics(ctx, next) {
+    const [namespace, method] = splitNamespace(ctx.method);
+    const scope = namespace ?? 'UnknownHandler';
+    const tracer = getTelemetryClient().getTracer(scope);
+    return tracer.startActiveSpan(
+      `${scope}.${method}`,
+      { kind: SpanKind.INTERNAL, attributes: { [ATTR_JSONRPC_METHOD]: ctx.method } },
+      async span => {
+        if (ctx.id !== null) {
+          span.setAttribute(ATTR_JSONRPC_REQUEST_ID, ctx.id);
+        }
+
+        try {
+          await next();
+          span.setStatus({ code: SpanStatusCode.OK });
+        } catch (err) {
+          span.setStatus({ code: SpanStatusCode.ERROR, message: err instanceof Error ? err.message : String(err) });
+          if (typeof err === 'string' || err instanceof Error) {
+            span.recordException(err);
+          }
+          throw err;
+        } finally {
+          span.end();
+        }
+      },
+    );
+  };
+}
+
+function splitNamespace(fullMethod: string): [namespace: string | undefined, method: string] {
+  const idx = fullMethod.indexOf('_');
+  if (idx > -1) {
+    return [fullMethod.slice(0, idx), fullMethod.slice(idx + 1)];
+  } else {
+    return [undefined, fullMethod];
+  }
+}

package/src/otel_resource.ts

@@ -7,10 +7,26 @@ import {
   osDetectorSync,
   serviceInstanceIdDetectorSync,
 } from '@opentelemetry/resources';
-import { SEMRESATTRS_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
+import { readFileSync } from 'fs';
+import { dirname, resolve } from 'path';
+import { fileURLToPath } from 'url';
 
 import { AZTEC_NODE_ROLE, AZTEC_REGISTRY_ADDRESS, AZTEC_ROLLUP_ADDRESS, AZTEC_ROLLUP_VERSION } from './attributes.js';
 
+/** Reads the Aztec client version from the release manifest. */
+function getAztecVersion(): string | undefined {
+  try {
+    const releasePleasePath = resolve(
+      dirname(fileURLToPath(import.meta.url)),
+      '../../../.release-please-manifest.json',
+    );
+    return JSON.parse(readFileSync(releasePleasePath, 'utf-8'))['.'];
+  } catch {
+    return undefined;
+  }
+}
+
 export function getOtelResource(): IResource {
   const resource = detectResourcesSync({
     detectors: [
@@ -42,7 +58,8 @@ const aztecNetworkDetectorSync: DetectorSync = {
     }
     const aztecAttributes = {
       // this gets overwritten by OTEL_RESOURCE_ATTRIBUTES (if set)
-      [SEMRESATTRS_SERVICE_NAME]: role ? `aztec-${role}` : undefined,
+      [ATTR_SERVICE_NAME]: role ? `aztec-${role}` : undefined,
+      [ATTR_SERVICE_VERSION]: getAztecVersion(),
       [AZTEC_NODE_ROLE]: role,
       [AZTEC_ROLLUP_VERSION]: process.env.ROLLUP_VERSION ?? 'canonical',
       [AZTEC_ROLLUP_ADDRESS]: process.env.ROLLUP_CONTRACT_ADDRESS,

package/src/prom_otel_adapter.ts

@@ -1,4 +1,4 @@
-import { type Logger, createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { Timer } from '@aztec/foundation/timer';
 
 import { Registry } from 'prom-client';
@@ -410,12 +410,11 @@ function parseLabelsSafely<Labels extends LabelsGeneric>(labelStr: string, logge
  */
 export class OtelMetricsAdapter extends Registry implements MetricsRegister {
   private readonly meter: Meter;
+  private logger: Logger;
 
-  constructor(
-    telemetryClient: TelemetryClient,
-    private logger: Logger = createLogger('telemetry:otel-metrics-adapter'),
-  ) {
+  constructor(telemetryClient: TelemetryClient, bindings?: LoggerBindings) {
     super();
+    this.logger = createLogger('telemetry:otel-metrics-adapter', bindings);
     this.meter = telemetryClient.getMeter('metrics-adapter');
   }
 

package/src/start.ts

@@ -1,4 +1,4 @@
-import { createLogger } from '@aztec/foundation/log';
+import { type LoggerBindings, createLogger } from '@aztec/foundation/log';
 
 import type { TelemetryClientConfig } from './config.js';
 import { NoopTelemetryClient } from './noop.js';
@@ -9,14 +9,22 @@ export * from './config.js';
 let initialized = false;
 let telemetry: TelemetryClient = new NoopTelemetryClient();
 
-export async function initTelemetryClient(config: TelemetryClientConfig): Promise<TelemetryClient> {
-  const log = createLogger('telemetry:client');
+export async function initTelemetryClient(
+  config: TelemetryClientConfig,
+  bindings?: LoggerBindings,
+): Promise<TelemetryClient> {
+  const log = createLogger('telemetry:client', bindings);
   if (initialized) {
     log.warn('Telemetry client has already been initialized once');
     return telemetry;
   }
 
-  if (config.metricsCollectorUrl || config.publicMetricsCollectorUrl) {
+  if (
+    config.metricsCollectorUrl ||
+    config.publicMetricsCollectorUrl ||
+    config.tracesCollectorUrl ||
+    config.logsCollectorUrl
+  ) {
     log.info(`Using OpenTelemetry client with custom collector`);
     // Lazy load OpenTelemetry to avoid loading heavy deps at startup
     const { OpenTelemetryClient } = await import('./otel.js');

package/src/telemetry.ts

@@ -20,7 +20,7 @@ import type * as Attributes from './attributes.js';
 import type { MetricDefinition } from './metrics.js';
 import { getTelemetryClient } from './start.js';
 
-export { toMetricOptions } from './metric-utils.js';
+export { toMetricOptions, createUpDownCounterWithDefault } from './metric-utils.js';
 
 export { type Span, SpanStatusCode, ValueType, type Context } from '@opentelemetry/api';
 
@@ -48,7 +48,6 @@ type BannedMetricAttributeNames = (typeof Attributes)[
   | 'TX_HASH'
   | 'PROVING_JOB_ID'
   | 'P2P_ID'
-  | 'P2P_REQ_RESP_BATCH_REQUESTS_COUNT'
   | 'TARGET_ADDRESS'
   | 'MANA_USED'
   | 'TOTAL_INSTRUCTIONS'];
@@ -56,8 +55,10 @@ type BannedMetricAttributeNames = (typeof Attributes)[
 /** Global registry of attributes */
 export type AttributesType = Partial<Record<AttributeNames, AttributeValue>>;
 
+export type AllowedAttributeNames = Exclude<AttributeNames, BannedMetricAttributeNames>;
+
 /** Subset of attributes allowed to be added to metrics */
-export type MetricAttributesType = Partial<Record<Exclude<AttributeNames, BannedMetricAttributeNames>, AttributeValue>>;
+export type MetricAttributesType = Partial<Record<AllowedAttributeNames, AttributeValue>>;
 
 /** Re-export MetricDefinition for convenience */
 export type { MetricDefinition } from './metrics.js';

package/src/wrappers/fetch.ts

@@ -9,12 +9,17 @@ import { ATTR_JSONRPC_METHOD, ATTR_JSONRPC_REQUEST_ID } from '../vendor/attribut
 
 /**
  * Makes a fetch function that retries based on the given attempts and propagates trace information.
- * @param retries - Sequence of intervals (in seconds) to retry.
+ * @param retries - Sequence of intervals (in seconds) to retry, or a factory function returning an iterator for custom/indefinite backoff.
  * @param noRetry - Whether to stop retries on server errors.
  * @param log - Optional logger for logging attempts.
  * @returns A fetch function.
  */
-export function makeTracedFetch(retries: number[], defaultNoRetry: boolean, fetch = defaultFetch, log?: Logger) {
+export function makeTracedFetch(
+  retries: number[] | (() => Generator<number>),
+  defaultNoRetry: boolean,
+  fetch = defaultFetch,
+  log?: Logger,
+) {
   return (host: string, body: unknown, extraHeaders: Record<string, string> = {}, noRetry?: boolean) => {
     const telemetry = getTelemetryClient();
     return telemetry.getTracer('fetch').startActiveSpan(`JsonRpcClient`, { kind: SpanKind.CLIENT }, async span => {
@@ -27,10 +32,11 @@ export function makeTracedFetch(retries: number[], defaultNoRetry: boolean, fetc
         }
         const headers = { ...extraHeaders };
         propagation.inject(context.active(), headers);
+        const backoff = typeof retries === 'function' ? retries() : makeBackoff(retries);
         return await retry(
           () => fetch(host, body, headers, noRetry ?? defaultNoRetry),
           `JsonRpcClient request to ${host}`,
-          makeBackoff(retries),
+          backoff,
           log,
           false,
         );

package/src/wrappers/index.ts

@@ -1,3 +1,2 @@
-export * from './l2_block_stream.js';
 export * from './fetch.js';
 export * from './json_rpc_server.js';

package/dest/wrappers/l2_block_stream.d.ts

@@ -1,16 +0,0 @@
-import { BlockNumber } from '@aztec/foundation/branded-types';
-import { type L2BlockSource, L2BlockStream, type L2BlockStreamEventHandler, type L2BlockStreamLocalDataProvider } from '@aztec/stdlib/block';
-import { type Traceable, type Tracer } from '@aztec/telemetry-client';
-/** Extends an L2BlockStream with a tracer to create a new trace per iteration. */
-export declare class TraceableL2BlockStream extends L2BlockStream implements Traceable {
-    readonly tracer: Tracer;
-    private readonly name;
-    constructor(l2BlockSource: Pick<L2BlockSource, 'getBlocks' | 'getBlockHeader' | 'getL2Tips' | 'getCheckpoints' | 'getCheckpointedBlocks'>, localData: L2BlockStreamLocalDataProvider, handler: L2BlockStreamEventHandler, tracer: Tracer, name?: string, log?: import("@aztec/foundation/log").Logger, opts?: {
-        proven?: boolean;
-        pollIntervalMS?: number;
-        batchSize?: number;
-        startingBlock?: BlockNumber;
-    });
-    work(): Promise<void>;
-}
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibDJfYmxvY2tfc3RyZWFtLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvd3JhcHBlcnMvbDJfYmxvY2tfc3RyZWFtLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUU5RCxPQUFPLEVBQ0wsS0FBSyxhQUFhLEVBQ2xCLGFBQWEsRUFDYixLQUFLLHlCQUF5QixFQUM5QixLQUFLLDhCQUE4QixFQUNwQyxNQUFNLHFCQUFxQixDQUFDO0FBQzdCLE9BQU8sRUFBRSxLQUFLLFNBQVMsRUFBRSxLQUFLLE1BQU0sRUFBYSxNQUFNLHlCQUF5QixDQUFDO0FBRWpGLGtGQUFrRjtBQUNsRixxQkFBYSxzQkFBdUIsU0FBUSxhQUFjLFlBQVcsU0FBUzthQVExRCxNQUFNLEVBQUUsTUFBTTtJQUM5QixPQUFPLENBQUMsUUFBUSxDQUFDLElBQUk7SUFSdkIsWUFDRSxhQUFhLEVBQUUsSUFBSSxDQUNqQixhQUFhLEVBQ2IsV0FBVyxHQUFHLGdCQUFnQixHQUFHLFdBQVcsR0FBRyxnQkFBZ0IsR0FBRyx1QkFBdUIsQ0FDMUYsRUFDRCxTQUFTLEVBQUUsOEJBQThCLEVBQ3pDLE9BQU8sRUFBRSx5QkFBeUIsRUFDbEIsTUFBTSxFQUFFLE1BQU0sRUFDYixJQUFJLEdBQUUsTUFBd0IsRUFDL0MsR0FBRyx5Q0FBcUMsRUFDeEMsSUFBSSxHQUFFO1FBQ0osTUFBTSxDQUFDLEVBQUUsT0FBTyxDQUFDO1FBQ2pCLGNBQWMsQ0FBQyxFQUFFLE1BQU0sQ0FBQztRQUN4QixTQUFTLENBQUMsRUFBRSxNQUFNLENBQUM7UUFDbkIsYUFBYSxDQUFDLEVBQUUsV0FBVyxDQUFDO0tBQ3hCLEVBR1A7SUFPUSxJQUFJLGtCQUVaO0NBQ0YifQ==

package/dest/wrappers/l2_block_stream.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"l2_block_stream.d.ts","sourceRoot":"","sources":["../../src/wrappers/l2_block_stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EACL,KAAK,aAAa,EAClB,aAAa,EACb,KAAK,yBAAyB,EAC9B,KAAK,8BAA8B,EACpC,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,MAAM,EAAa,MAAM,yBAAyB,CAAC;AAEjF,kFAAkF;AAClF,qBAAa,sBAAuB,SAAQ,aAAc,YAAW,SAAS;aAQ1D,MAAM,EAAE,MAAM;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI;IARvB,YACE,aAAa,EAAE,IAAI,CACjB,aAAa,EACb,WAAW,GAAG,gBAAgB,GAAG,WAAW,GAAG,gBAAgB,GAAG,uBAAuB,CAC1F,EACD,SAAS,EAAE,8BAA8B,EACzC,OAAO,EAAE,yBAAyB,EAClB,MAAM,EAAE,MAAM,EACb,IAAI,GAAE,MAAwB,EAC/C,GAAG,yCAAqC,EACxC,IAAI,GAAE;QACJ,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,WAAW,CAAC;KACxB,EAGP;IAOQ,IAAI,kBAEZ;CACF"}