@aztec/telemetry-client 0.0.1-commit.f504929 → 0.0.1-commit.f5a9928

package/src/metrics.ts

@@ -33,6 +33,46 @@ export interface MetricDefinition {
   readonly valueType?: ValueType;
 }
 
+export const JSON_RPC_SERVER_REQUEST_COUNT: MetricDefinition = {
+  name: 'aztec.json_rpc.server.request_count',
+  description: 'Number of completed JSON-RPC server requests',
+  valueType: ValueType.INT,
+};
+export const JSON_RPC_SERVER_REQUEST_DURATION: MetricDefinition = {
+  name: 'aztec.json_rpc.server.request_duration',
+  description: 'JSON-RPC server request handler duration',
+  unit: 'ms',
+  valueType: ValueType.DOUBLE,
+};
+export const JSON_RPC_SERVER_REQUEST_VALIDATION_DURATION: MetricDefinition = {
+  name: 'aztec.json_rpc.server.request_validation_duration',
+  description: 'JSON-RPC server request parameter validation duration',
+  unit: 'ms',
+  valueType: ValueType.DOUBLE,
+};
+export const JSON_RPC_SERVER_REJECTED_REQUEST_COUNT: MetricDefinition = {
+  name: 'aztec.json_rpc.server.rejected_request_count',
+  description: 'Number of JSON-RPC requests rejected before handler dispatch',
+  valueType: ValueType.INT,
+};
+export const JSON_RPC_SERVER_BATCH_COUNT: MetricDefinition = {
+  name: 'aztec.json_rpc.server.batch_count',
+  description: 'Number of JSON-RPC batches received',
+  valueType: ValueType.INT,
+};
+export const JSON_RPC_SERVER_BATCH_DURATION: MetricDefinition = {
+  name: 'aztec.json_rpc.server.batch_duration',
+  description: 'JSON-RPC batch processing duration',
+  unit: 'ms',
+  valueType: ValueType.DOUBLE,
+};
+export const JSON_RPC_SERVER_BATCH_SIZE: MetricDefinition = {
+  name: 'aztec.json_rpc.server.batch_size',
+  description: 'Number of requests in a JSON-RPC batch',
+  unit: 'requests',
+  valueType: ValueType.INT,
+};
+
 export const BLOB_SINK_STORE_REQUESTS: MetricDefinition = {
   name: 'aztec.blob_sink.store_request_count',
   description: 'Number of blob store requests',
@@ -162,7 +202,7 @@ export const MEMPOOL_TX_ADDED_COUNT: MetricDefinition = {
 };
 export const MEMPOOL_TX_MINED_DELAY: MetricDefinition = {
   name: 'aztec.mempool.tx_mined_delay',
-  description: 'Delay between transaction added and evicted from the mempool',
+  description: 'Delay (ms) from a transaction being received into the pool to being mined',
   unit: 'ms',
   valueType: ValueType.INT,
 };
@@ -204,6 +244,24 @@ export const MEMPOOL_TX_POOL_V2_METADATA_MEMORY: MetricDefinition = {
   valueType: ValueType.INT,
 };
 
+export const MEMPOOL_TX_POOL_V2_QUEUE_WAIT: MetricDefinition = {
+  name: 'aztec.mempool.tx_pool_v2.queue_wait',
+  description: 'Time an operation waits in the tx pool serial queue before executing, keyed by mempool.operation',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const MEMPOOL_TX_POOL_V2_QUEUE_EXECUTION: MetricDefinition = {
+  name: 'aztec.mempool.tx_pool_v2.queue_execution',
+  description: 'Time an operation spends executing in the tx pool serial queue, keyed by mempool.operation',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const MEMPOOL_TX_POOL_V2_QUEUE_LENGTH: MetricDefinition = {
+  name: 'aztec.mempool.tx_pool_v2.queue_length',
+  description: 'Number of operations waiting in the tx pool serial queue',
+  valueType: ValueType.INT,
+};
+
 export const MEMPOOL_TX_POOL_V2_DUPLICATE_ADD: MetricDefinition = {
   name: 'aztec.mempool.tx_pool_v2.duplicate_add',
   description: 'Transactions received via addPendingTxs that were already in the pool',
@@ -308,6 +366,12 @@ export const ARCHIVER_SYNC_PER_BLOCK: MetricDefinition = {
   unit: 'ms',
   valueType: ValueType.INT,
 };
+export const ARCHIVER_SYNC_PER_CHECKPOINT: MetricDefinition = {
+  name: 'aztec.archiver.checkpoint.sync_per_item_duration',
+  description: 'Duration to sync a checkpoint',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
 export const ARCHIVER_SYNC_BLOCK_COUNT: MetricDefinition = {
   name: 'aztec.archiver.block.sync_count',
   description: 'Number of blocks synced from L1',
@@ -334,7 +398,8 @@ export const ARCHIVER_PRUNE_DURATION: MetricDefinition = {
 };
 export const ARCHIVER_PRUNE_COUNT: MetricDefinition = {
   name: 'aztec.archiver.prune_count',
-  description: 'Number of prunes detected',
+  description:
+    'Number of prunes detected, dimensioned by prune_type: unproven (epoch prune of checkpoints that will not be proven), uncheckpointed (proposed blocks whose slot ended without a checkpoint), l1_conflict (proposed blocks conflicting with an L1 checkpoint), orphan (proposed blocks whose matching proposed checkpoint never arrived before the deadline), and l1_mismatch (the local checkpointed tip diverged from L1 — an L1 reorg or a pruned/missed-proof checkpoint — rewinding already-checkpointed blocks).',
   valueType: ValueType.INT,
 };
 
@@ -357,6 +422,12 @@ export const ARCHIVER_CHECKPOINT_L1_INCLUSION_DELAY: MetricDefinition = {
   valueType: ValueType.INT,
 };
 
+export const ARCHIVER_CHECKPOINT_PROMOTED_COUNT: MetricDefinition = {
+  name: 'aztec.archiver.checkpoint_promoted_count',
+  description: 'Number of checkpoints promoted from proposed (blob fetch skipped)',
+  valueType: ValueType.INT,
+};
+
 export const NODE_RECEIVE_TX_DURATION: MetricDefinition = {
   name: 'aztec.node.receive_tx.duration',
   description: 'The duration of the receiveTx method',
@@ -388,6 +459,12 @@ export const SEQUENCER_STATE_TRANSITION_BUFFER_DURATION: MetricDefinition = {
   unit: 'ms',
   valueType: ValueType.INT,
 };
+export const SEQUENCER_STATE_DURATION: MetricDefinition = {
+  name: 'aztec.sequencer.state_duration',
+  description: 'Wall-clock time spent in each sequencer state, labelled by the state being left',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
 export const SEQUENCER_BLOCK_BUILD_DURATION: MetricDefinition = {
   name: 'aztec.sequencer.block.build_duration',
   description: 'Duration to build a block',
@@ -405,6 +482,12 @@ export const SEQUENCER_BLOCK_COUNT: MetricDefinition = {
   description: 'Number of blocks built by this sequencer',
   valueType: ValueType.INT,
 };
+export const SEQUENCER_BLOCK_INTER_BLOCK_TIME: MetricDefinition = {
+  name: 'aztec.sequencer.block.inter_block_time',
+  description: 'Wall-clock time elapsed between consecutive blocks being built by this sequencer',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
 export const SEQUENCER_CURRENT_SLOT_REWARDS: MetricDefinition = {
   name: 'aztec.sequencer.current_slot_rewards',
   description: 'The rewards earned per filled slot',
@@ -477,6 +560,25 @@ export const SEQUENCER_CHECKPOINT_BUILD_DURATION: MetricDefinition = {
   unit: 'ms',
   valueType: ValueType.INT,
 };
+export const SEQUENCER_CHECKPOINT_START_TO_FIRST_BLOCK_DURATION: MetricDefinition = {
+  name: 'aztec.sequencer.checkpoint.start_to_first_block_duration',
+  description: 'Time from starting checkpoint work to the first block finishing build',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const SEQUENCER_CHECKPOINT_LAST_BLOCK_TO_BROADCAST_DURATION: MetricDefinition = {
+  name: 'aztec.sequencer.checkpoint.last_block_to_broadcast_duration',
+  description: 'Time from the final block finishing build to the checkpoint proposal being broadcast',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const SEQUENCER_PIPELINED_CHECKPOINT_BUILD_START_OFFSET_FROM_SLOT_BOUNDARY: MetricDefinition = {
+  name: 'aztec.sequencer.pipelined_checkpoint.build_start_offset_from_slot_boundary',
+  description:
+    'Absolute offset from the wall-clock slot boundary when a pipelined checkpoint build starts. Use aztec.slot_boundary_side to distinguish before vs after the boundary.',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
 export const SEQUENCER_CHECKPOINT_BLOCK_COUNT: MetricDefinition = {
   name: 'aztec.sequencer.checkpoint.block_count',
   description: 'Number of blocks built in a checkpoint',
@@ -499,11 +601,66 @@ export const SEQUENCER_SLASHING_ATTEMPTS_COUNT: MetricDefinition = {
   description: 'The number of slashing action attempts',
   valueType: ValueType.INT,
 };
+export const SLASHER_ROUND_EXECUTED_COUNT: MetricDefinition = {
+  name: 'aztec.slasher.round.executed_count',
+  description: 'The number of slashing rounds executed',
+  valueType: ValueType.INT,
+};
+export const SLASHER_OWN_VALIDATOR_TARGETED_COUNT: MetricDefinition = {
+  name: 'aztec.slasher.own_validator.targeted_count',
+  description:
+    "The number of times one of the node's own validators was named as a slash target by an onchain vote, counted " +
+    'once per vote per validator. Votes cast while the node is offline or starting up are not counted, and L1 ' +
+    'reorgs can cause small drift within a round.',
+  valueType: ValueType.INT,
+};
+export const SLASHER_OWN_VALIDATOR_CURRENT_ROUND_VOTES_MAX: MetricDefinition = {
+  name: 'aztec.slasher.own_validator.current_round_votes_max',
+  description:
+    "Highest number of votes cast against any committee position held by the node's own validators in the current " +
+    'slashing round, to compare against aztec.slasher.quorum_size (the contract tallies quorum per position). ' +
+    'Resets to zero when a new round starts. Votes cast while the node is offline or starting up are not counted, ' +
+    'and L1 reorgs can cause small drift within a round.',
+  valueType: ValueType.INT,
+};
+export const SLASHER_QUORUM_SIZE: MetricDefinition = {
+  name: 'aztec.slasher.quorum_size',
+  description: 'The number of votes a validator must receive in a round to be slashed',
+  valueType: ValueType.INT,
+};
+export const SLASHER_OWN_VALIDATOR_SLASHED_COUNT: MetricDefinition = {
+  name: 'aztec.slasher.own_validator.slashed_count',
+  description: "The number of times one of the node's own validators was slashed, from executed Slashed events",
+  valueType: ValueType.INT,
+};
+export const SLASHER_OWN_VALIDATOR_SLASHED_AMOUNT: MetricDefinition = {
+  name: 'aztec.slasher.own_validator.slashed_amount',
+  description:
+    "Cumulative amount slashed from the node's own validators in whole staking-asset tokens, from executed " +
+    'Slashed events',
+  unit: 'tokens',
+  valueType: ValueType.DOUBLE,
+};
 export const SEQUENCER_CHECKPOINT_SUCCESS_COUNT: MetricDefinition = {
   name: 'aztec.sequencer.checkpoint.success_count',
   description: 'The number of times checkpoint publishing succeeded',
   valueType: ValueType.INT,
 };
+export const SEQUENCER_PIPELINE_DEPTH: MetricDefinition = {
+  name: 'aztec.sequencer.pipeline.depth',
+  description: 'Current pipeline depth when builder pipelining is enabled',
+  valueType: ValueType.INT,
+};
+export const SEQUENCER_PIPELINE_DISCARDS_COUNT: MetricDefinition = {
+  name: 'aztec.sequencer.pipeline.discards_count',
+  description: 'The number of times a pipeline was discarded',
+  valueType: ValueType.INT,
+};
+export const SEQUENCER_PIPELINE_PARENT_CHECKPOINT_MISMATCH_COUNT: MetricDefinition = {
+  name: 'aztec.sequencer.pipeline.parent_checkpoint_mismatch_count',
+  description: 'The number of times a pipelined checkpoint was discarded because the parent did not match expectations',
+  valueType: ValueType.INT,
+};
 
 // Fisherman fee analysis metrics
 export const FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED: MetricDefinition = {
@@ -671,6 +828,24 @@ export const L1_PUBLISHER_TX_TOTAL_FEE: MetricDefinition = {
   unit: 'eth',
   valueType: ValueType.DOUBLE,
 };
+export const PROVER_NODE_ESTIMATED_SUBMISSION_GAS: MetricDefinition = {
+  name: 'aztec.prover_node.estimated_submission.gas',
+  description: 'Estimated gas for a proof submission tx (proof publishing disabled, not actually sent)',
+  unit: 'gas',
+  valueType: ValueType.INT,
+};
+export const PROVER_NODE_ESTIMATED_SUBMISSION_GAS_PRICE: MetricDefinition = {
+  name: 'aztec.prover_node.estimated_submission.gas_price',
+  description: 'Estimated effective gas price for a proof submission tx (proof publishing disabled, not actually sent)',
+  unit: 'gwei',
+  valueType: ValueType.DOUBLE,
+};
+export const PROVER_NODE_ESTIMATED_SUBMISSION_TOTAL_FEE: MetricDefinition = {
+  name: 'aztec.prover_node.estimated_submission.total_fee',
+  description: 'Estimated total L1 fee for a proof submission tx (proof publishing disabled, not actually sent)',
+  unit: 'eth',
+  valueType: ValueType.DOUBLE,
+};
 
 export const L1_BLOCK_HEIGHT: MetricDefinition = {
   name: 'aztec.l1.block_height',
@@ -899,6 +1074,19 @@ export const P2P_GOSSIP_AGG_MESSAGE_VALIDATION_DURATION_AVG: MetricDefinition =
   valueType: ValueType.INT,
 };
 
+export const P2P_GOSSIP_SLOW_VALIDATION_COUNT: MetricDefinition = {
+  name: 'aztec.p2p.gossip.slow_validation_count',
+  description: 'Number of gossip validations that exceeded 75% of the mcache eviction window',
+  valueType: ValueType.INT,
+};
+
+export const P2P_GOSSIP_TX_VALIDATION_STAGE_DURATION: MetricDefinition = {
+  name: 'aztec.p2p.gossip.tx_validation_stage_duration',
+  description: 'Duration of each stage of gossiped tx validation, keyed by the tx_validation_stage attribute',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+
 export const PUBLIC_PROCESSOR_TX_DURATION: MetricDefinition = {
   name: 'aztec.public_processor.tx_duration',
   description: 'Duration to process a public transaction',
@@ -962,6 +1150,17 @@ export const PUBLIC_PROCESSOR_TREE_INSERTION: MetricDefinition = {
   unit: 'ms',
   valueType: ValueType.INT,
 };
+export const PUBLIC_PROCESSOR_SILENTLY_SKIPPED_COUNT: MetricDefinition = {
+  name: 'aztec.public_processor.silently_skipped_count',
+  description: 'Public txs fully processed then skipped (e.g. blob-field limit); not counted as processed or failed',
+  valueType: ValueType.INT,
+};
+export const PUBLIC_PROCESSOR_SILENTLY_SKIPPED_DURATION: MetricDefinition = {
+  name: 'aztec.public_processor.silently_skipped_duration',
+  description: 'Wall-clock time spent processing txs that were then silently skipped',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
 
 export const PUBLIC_EXECUTOR_PREFIX = 'aztec.public_executor.';
 export const PUBLIC_EXECUTOR_SIMULATION_COUNT: MetricDefinition = {
@@ -1106,12 +1305,6 @@ export const PROVING_AGENT_IDLE: MetricDefinition = {
   valueType: ValueType.DOUBLE,
 };
 
-export const PROVER_NODE_EXECUTION_DURATION: MetricDefinition = {
-  name: 'aztec.prover_node.execution.duration',
-  description: 'Duration of execution of an epoch by the prover',
-  unit: 'ms',
-  valueType: ValueType.INT,
-};
 export const PROVER_NODE_JOB_DURATION: MetricDefinition = {
   name: 'aztec.prover_node.job_duration',
   description: 'Duration of proving job',
@@ -1133,6 +1326,51 @@ export const PROVER_NODE_JOB_TRANSACTIONS: MetricDefinition = {
   description: 'Number of transactions in a proven epoch',
   valueType: ValueType.INT,
 };
+export const PROVER_NODE_BLOB_PROCESSING_LAST_DURATION: MetricDefinition = {
+  name: 'aztec.prover_node.blob_processing.last_duration',
+  description: 'Duration of blob processing step in epoch proving job',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const PROVER_NODE_BLOCK_PROCESSING_DURATION: MetricDefinition = {
+  name: 'aztec.prover_node.block_processing.duration',
+  description: 'Duration of processing a single block in epoch proving job',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const PROVER_NODE_CHECKPOINT_PROCESSING_DURATION: MetricDefinition = {
+  name: 'aztec.prover_node.checkpoint_processing.duration',
+  description: 'Duration of processing a single checkpoint in epoch proving job',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const PROVER_NODE_CHECKPOINT_BLOCKS: MetricDefinition = {
+  name: 'aztec.prover_node.checkpoint_blocks',
+  description: 'Number of blocks in a proven checkpoint',
+  valueType: ValueType.INT,
+};
+export const PROVER_NODE_CHECKPOINT_TRANSACTIONS: MetricDefinition = {
+  name: 'aztec.prover_node.checkpoint_transactions',
+  description: 'Number of transactions in a proven checkpoint',
+  valueType: ValueType.INT,
+};
+export const PROVER_NODE_CHECKPOINT_PROVING_DURATION: MetricDefinition = {
+  name: 'aztec.prover_node.checkpoint_proving.duration',
+  description:
+    'Duration from the start of checkpoint processing to its block proofs being ready (excludes tx gathering)',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const PROVER_NODE_ACTIVE_CHECKPOINTS: MetricDefinition = {
+  name: 'aztec.prover_node.active_checkpoints',
+  description: 'Current number of canonical CheckpointProvers in the store (i.e. checkpoints currently being proven)',
+  valueType: ValueType.INT,
+};
+export const PROVER_NODE_ACTIVE_EPOCH_SESSIONS: MetricDefinition = {
+  name: 'aztec.prover_node.active_epoch_sessions',
+  description: 'Current number of live EpochSessions, broken down by kind (full|partial)',
+  valueType: ValueType.INT,
+};
 export const PROVER_NODE_REWARDS_TOTAL: MetricDefinition = {
   name: 'aztec.prover_node.rewards_total',
   description: 'The rewards earned (total)',
@@ -1246,6 +1484,19 @@ export const VALIDATOR_RE_EXECUTION_TX_COUNT: MetricDefinition = {
   unit: 'tx',
   valueType: ValueType.INT,
 };
+export const VALIDATOR_CHECKPOINT_PROPOSAL_TO_PIPELINED_STATE_DURATION: MetricDefinition = {
+  name: 'aztec.validator.checkpoint_proposal_to_pipelined_state_duration',
+  description: 'Time from receiving a checkpoint proposal to setting proposed checkpoint state for pipelining',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const VALIDATOR_CHECKPOINT_PROPOSAL_RECEIVE_OFFSET_FROM_NEXT_SLOT_BOUNDARY: MetricDefinition = {
+  name: 'aztec.validator.checkpoint.proposal_receive_offset_from_next_slot_boundary',
+  description:
+    'Absolute offset from the next slot boundary when a foreign checkpoint proposal is received. Use aztec.slot_boundary_side to distinguish before vs after the boundary.',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
 
 export const VALIDATOR_FAILED_REEXECUTION_COUNT: MetricDefinition = {
   name: 'aztec.validator.failed_reexecution_count',

package/src/monitored_batch_span_processor.ts

@@ -0,0 +1,100 @@
+import type { Logger } from '@aztec/foundation/log';
+
+import { type Context, SpanStatusCode } from '@opentelemetry/api';
+import { hrTimeToMilliseconds } from '@opentelemetry/core';
+import type { SpanExporter } from '@opentelemetry/sdk-trace-base';
+import { BatchSpanProcessor, type BufferConfig, type ReadableSpan, type Span } from '@opentelemetry/sdk-trace-node';
+
+/** Minimum interval between drop warnings to avoid log spam. */
+const DROP_WARNING_INTERVAL_MS = 30_000;
+
+const DEFAULT_MIN_TRACE_DURATION_MS = 10;
+
+const DEFAULT_MAX_QUEUE_SIZE = 16384;
+
+/** Cap on the per-export batch size, so a large queue can actually be drained instead of dribbling out
+ * at the SDK default of 512 spans per scheduled export. Kept <= maxQueueSize per the BatchSpanProcessor contract. */
+const DEFAULT_MAX_EXPORT_BATCH_SIZE = 2048;
+
+export type MonitoredBatchSpanProcessorConfig = BufferConfig & {
+  minTraceDurationMs?: number;
+};
+
+/**
+ * Wraps BatchSpanProcessor to emit warnings when spans are dropped due to a full queue.
+ * The standard BatchSpanProcessor silently discards spans when its internal queue reaches
+ * maxQueueSize, making telemetry data loss invisible to operators.
+ */
+export class MonitoredBatchSpanProcessor extends BatchSpanProcessor {
+  private readonly maxQueueSize: number;
+  private readonly minTraceDurationMs: number;
+  private readonly log: Logger;
+
+  private approxQueueSize = 0;
+  private droppedSinceLastWarning = 0;
+  private totalDropped = 0;
+  private lastWarningTime = 0;
+
+  constructor(exporter: SpanExporter, log: Logger, config?: MonitoredBatchSpanProcessorConfig) {
+    const maxQueueSize = config?.maxQueueSize ?? DEFAULT_MAX_QUEUE_SIZE;
+    const maxExportBatchSize = Math.min(config?.maxExportBatchSize ?? DEFAULT_MAX_EXPORT_BATCH_SIZE, maxQueueSize);
+    super(exporter, { ...config, maxQueueSize, maxExportBatchSize });
+    this.maxQueueSize = maxQueueSize;
+    this.minTraceDurationMs = Math.max(0, config?.minTraceDurationMs ?? DEFAULT_MIN_TRACE_DURATION_MS);
+    this.log = log;
+  }
+
+  override onStart(span: Span, parentContext: Context): void {
+    super.onStart(span, parentContext);
+  }
+
+  override onEnd(span: ReadableSpan): void {
+    if (this.shouldDropShortSpan(span)) {
+      return;
+    }
+
+    if (this.approxQueueSize >= this.maxQueueSize) {
+      this.droppedSinceLastWarning++;
+      this.totalDropped++;
+      this.maybeLogDropWarning();
+    } else {
+      this.approxQueueSize++;
+    }
+    super.onEnd(span);
+  }
+
+  override async forceFlush(): Promise<void> {
+    await super.forceFlush();
+    this.approxQueueSize = 0;
+  }
+
+  override async shutdown(): Promise<void> {
+    if (this.totalDropped > 0) {
+      this.log.warn(`BatchSpanProcessor shutting down with ${this.totalDropped} total spans dropped`, {
+        totalDropped: this.totalDropped,
+      });
+    }
+    await super.shutdown();
+  }
+
+  private shouldDropShortSpan(span: ReadableSpan): boolean {
+    return (
+      this.minTraceDurationMs > 0 &&
+      span.status.code !== SpanStatusCode.ERROR &&
+      hrTimeToMilliseconds(span.duration) < this.minTraceDurationMs
+    );
+  }
+
+  private maybeLogDropWarning(): void {
+    const now = Date.now();
+    if (now - this.lastWarningTime >= DROP_WARNING_INTERVAL_MS) {
+      this.log.warn(
+        `BatchSpanProcessor dropping spans: queue full (maxQueueSize=${this.maxQueueSize}). ` +
+          `${this.droppedSinceLastWarning} dropped since last warning, ${this.totalDropped} total.`,
+        { droppedSinceLastWarning: this.droppedSinceLastWarning, totalDropped: this.totalDropped },
+      );
+      this.droppedSinceLastWarning = 0;
+      this.lastWarningTime = now;
+    }
+  }
+}

package/src/otel.ts

@@ -28,12 +28,13 @@ import {
   type PeriodicExportingMetricReaderOptions,
   View,
 } from '@opentelemetry/sdk-metrics';
-import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
 import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
 
 import type { TelemetryClientConfig } from './config.js';
 import { toMetricOptions } from './metric-utils.js';
 import type { MetricDefinition } from './metrics.js';
+import { MonitoredBatchSpanProcessor } from './monitored_batch_span_processor.js';
 import { NodejsMetricsMonitor } from './nodejs_metrics_monitor.js';
 import { OtelFilterMetricExporter, PublicOtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
 import { registerOtelLoggerProvider } from './otel_logger_provider.js';
@@ -95,6 +96,11 @@ export class OpenTelemetryClient implements TelemetryClient {
   private meters: Map<string, WrappedMeter> = new Map<string, WrappedMeter>();
   private tracers: Map<string, Tracer> = new Map<string, Tracer>();
 
+  /** Memoized shutdown promise. The telemetry client is shared between the aztec-node and an embedded prover-node,
+   * so stop() can be invoked more than once; the providers throw "shutdown may only be called once" and
+   * "invalid attempt to force flush after shutdown" if that happens. Guarding here makes stop()/flush() idempotent. */
+  private stopPromise: Promise<void> | undefined;
+
   protected constructor(
     private resource: IResource,
     private meterProvider: MeterProvider,
@@ -168,6 +174,10 @@ export class OpenTelemetryClient implements TelemetryClient {
   }
 
   public async flush() {
+    // Flushing after the providers have been shut down throws "invalid attempt to force flush after shutdown".
+    if (this.stopPromise) {
+      return;
+    }
     await Promise.all([
       this.meterProvider.forceFlush(),
       this.loggerProvider?.forceFlush(),
@@ -175,7 +185,11 @@ export class OpenTelemetryClient implements TelemetryClient {
     ]);
   }
 
-  public async stop() {
+  public stop() {
+    return (this.stopPromise ??= this.doStop());
+  }
+
+  private async doStop() {
     this.nodejsMetricsMonitor?.stop();
 
     const flushAndShutdown = async (provider?: { forceFlush: () => Promise<void>; shutdown: () => Promise<void> }) => {
@@ -241,6 +255,23 @@ export class OpenTelemetryClient implements TelemetryClient {
             true,
           ),
         }),
+        // Pending-to-mined delay routinely exceeds the 1-minute ceiling of the generic `ms`
+        // view below under load, so it would saturate at 60s. Give this one metric wider
+        // buckets (1s to 10min). This must precede the generic `ms` view: when multiple views
+        // match an instrument, the SDK keeps the first-registered compatible storage, so the
+        // first view in this list wins the bucket boundaries.
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentName: 'aztec.mempool.tx_mined_delay',
+          instrumentUnit: 'ms',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [
+              1_000, 2_500, 5_000, 7_500, 10_000, 15_000, 30_000, 45_000, 60_000, 90_000, 120_000, 180_000, 300_000,
+              600_000,
+            ],
+            true,
+          ),
+        }),
         new View({
           instrumentType: InstrumentType.HISTOGRAM,
           instrumentUnit: 'ms',
@@ -334,6 +365,36 @@ export class OpenTelemetryClient implements TelemetryClient {
             true,
           ),
         }),
+        // L1 gas prices in gwei: priority fees ~0.01-10, base fees ~1-500, spikes to 1000+
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'gwei',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1_000],
+            true,
+          ),
+        }),
+        // L1 gas consumption: tx gas 100k-30M, calldata/blob gas varies
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'gas',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [
+              10_000, 50_000, 100_000, 250_000, 500_000, 1_000_000, 2_000_000, 5_000_000, 10_000_000, 15_000_000,
+              30_000_000,
+            ],
+            true,
+          ),
+        }),
+        // L1 tx total fee in ETH: typically 0.001 - 1 ETH
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'eth',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10],
+            true,
+          ),
+        }),
       ],
     });
   }
@@ -343,7 +404,12 @@ export class OpenTelemetryClient implements TelemetryClient {
       const tracerProvider = new NodeTracerProvider({
         resource,
         spanProcessors: config.tracesCollectorUrl
-          ? [new BatchSpanProcessor(new OTLPTraceExporter({ url: config.tracesCollectorUrl.href }))]
+          ? [
+              new MonitoredBatchSpanProcessor(new OTLPTraceExporter({ url: config.tracesCollectorUrl.href }), log, {
+                maxQueueSize: config.otelBspMaxQueueSize,
+                minTraceDurationMs: config.otelMinTraceDurationMs,
+              }),
+            ]
           : [],
       });