@aztec/telemetry-client 0.0.1-commit.42ee6df9b → 0.0.1-commit.431c48d

package/src/attributes.ts

@@ -18,6 +18,8 @@
 export const HTTP_REQUEST_HOST = 'http.header.request.host';
 export const HTTP_RESPONSE_STATUS_CODE = 'http.response.status_code';
 
+export const JSON_RPC_REJECTION_REASON = 'aztec.json_rpc.rejection_reason';
+
 /** The Aztec network identifier */
 export const NETWORK_NAME = 'aztec.network_name';
 
@@ -45,6 +47,8 @@ export const BLOCK_NUMBER = 'aztec.block.number';
 export const BLOCK_HASH = 'aztec.block.hash';
 /** The slot number */
 export const SLOT_NUMBER = 'aztec.slot.number';
+/** Whether an event happened before or after a slot boundary. */
+export const SLOT_BOUNDARY_SIDE = 'aztec.slot_boundary_side';
 /** The checkpoint number */
 export const CHECKPOINT_NUMBER = 'aztec.checkpoint.number';
 /** The parent's block number */
@@ -61,6 +65,8 @@ export const EPOCH_SIZE = 'aztec.epoch.size';
 export const BLOCK_PROPOSER = 'aztec.block.proposer';
 /** The epoch number */
 export const EPOCH_NUMBER = 'aztec.epoch.number';
+/** Kind of an EpochSession: 'full' or 'partial'. */
+export const EPOCH_SESSION_KIND = 'aztec.epoch_session.kind';
 /** The tx hash */
 export const TX_HASH = 'aztec.tx.hash';
 /** Generic attribute representing whether the action was successful or not */
@@ -69,6 +75,8 @@ export const OK = 'aztec.ok';
 export const STATUS = 'aztec.status';
 /** Generic error type attribute */
 export const ERROR_TYPE = 'aztec.error_type';
+/** The cause of an archiver prune (unproven / uncheckpointed / l1_conflict / orphan / l1_mismatch) */
+export const PRUNE_TYPE = 'aztec.archiver.prune_type';
 /** The type of the transaction */
 export const L1_TX_TYPE = 'aztec.l1.tx_type';
 /** The L1 address of the entity that sent a transaction to L1 */
@@ -94,7 +102,6 @@ export const VALIDATOR_STATUS = 'aztec.validator_status';
 
 export const P2P_ID = 'aztec.p2p.id';
 export const P2P_REQ_RESP_PROTOCOL = 'aztec.p2p.req_resp.protocol';
-export const P2P_REQ_RESP_BATCH_REQUESTS_COUNT = 'aztec.p2p.req_resp.batch_requests_count';
 /** The state of a peer (Healthy, Disconnect, Banned) */
 export const P2P_PEER_SCORE_STATE = 'aztec.p2p.peer_score_state';
 export const POOL_NAME = 'aztec.pool.name';
@@ -128,6 +135,12 @@ export const NODEJS_EVENT_LOOP_STATE = 'nodejs.eventloop.state';
 
 export const TOPIC_NAME = 'aztec.gossip.topic_name';
 
+/** Stage of gossiped tx validation (deserialize, fast_validation_setup, fast_validation, pool_precheck, proof_verify, pool_add) */
+export const TX_VALIDATION_STAGE = 'aztec.p2p.tx_validation_stage';
+
+/** Operation type enqueued on the mempool serial queue */
+export const MEMPOOL_OPERATION = 'aztec.mempool.operation';
+
 /** The reason a transaction was evicted from the tx pool */
 export const TX_POOL_EVICTION_REASON = 'aztec.mempool.eviction_reason';
 

package/src/config.ts

@@ -1,4 +1,9 @@
-import { type ConfigMappingsType, booleanConfigHelper, getConfigFromMappings } from '@aztec/foundation/config';
+import {
+  type ConfigMappingsType,
+  booleanConfigHelper,
+  getConfigFromMappings,
+  numberConfigHelper,
+} from '@aztec/foundation/config';
 
 export interface TelemetryClientConfig {
   metricsCollectorUrl?: URL;
@@ -12,35 +17,35 @@ export interface TelemetryClientConfig {
   otelExportTimeoutMs: number;
   otelExcludeMetrics: string[];
   otelIncludeMetrics: string[];
+  otelMinTraceDurationMs: number;
+  otelBspMaxQueueSize: number;
 }
 
 export const telemetryClientConfigMappings: ConfigMappingsType<TelemetryClientConfig> = {
   metricsCollectorUrl: {
     env: 'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT',
     description: 'The URL of the telemetry collector for metrics',
-    parseEnv: (val: string) => val && new URL(val),
+    parseEnv: (val: string) => new URL(val),
   },
   tracesCollectorUrl: {
     env: 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT',
     description: 'The URL of the telemetry collector for traces',
-    parseEnv: (val: string) => val && new URL(val),
+    parseEnv: (val: string) => new URL(val),
   },
   logsCollectorUrl: {
     env: 'OTEL_EXPORTER_OTLP_LOGS_ENDPOINT',
     description: 'The URL of the telemetry collector for logs',
-    parseEnv: (val: string) => val && new URL(val),
+    parseEnv: (val: string) => new URL(val),
   },
   otelCollectIntervalMs: {
     env: 'OTEL_COLLECT_INTERVAL_MS',
     description: 'The interval at which to collect metrics',
-    defaultValue: 60000, // Default extracted from otel client
-    parseEnv: (val: string) => parseInt(val),
+    ...numberConfigHelper(60000),
   },
   otelExportTimeoutMs: {
     env: 'OTEL_EXPORT_TIMEOUT_MS',
     description: 'The timeout for exporting metrics',
-    defaultValue: 30000, // Default extracted from otel client
-    parseEnv: (val: string) => parseInt(val),
+    ...numberConfigHelper(30000),
   },
   otelExcludeMetrics: {
     env: 'OTEL_EXCLUDE_METRICS',
@@ -54,6 +59,16 @@ export const telemetryClientConfigMappings: ConfigMappingsType<TelemetryClientCo
         : [],
     defaultValue: [],
   },
+  otelMinTraceDurationMs: {
+    env: 'OTEL_MIN_TRACE_DURATION_MS',
+    description: 'The minimum successful trace duration to export in milliseconds. Set to 0 to export all traces.',
+    ...numberConfigHelper(10),
+  },
+  otelBspMaxQueueSize: {
+    env: 'OTEL_BSP_MAX_QUEUE_SIZE',
+    description: 'The maximum number of completed spans to queue before export.',
+    ...numberConfigHelper(16384),
+  },
   otelIncludeMetrics: {
     env: 'OTEL_INCLUDE_METRICS',
     description: 'A list of metric prefixes to include in export (ignored if OTEL_EXCLUDE_METRICS is set)',
@@ -70,7 +85,7 @@ export const telemetryClientConfigMappings: ConfigMappingsType<TelemetryClientCo
   publicMetricsCollectorUrl: {
     env: 'PUBLIC_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT',
     description: 'A URL to publish a subset of metrics for public consumption',
-    parseEnv: (val: string) => val && new URL(val),
+    parseEnv: (val: string) => new URL(val),
   },
   publicMetricsCollectFrom: {
     env: 'PUBLIC_OTEL_COLLECT_FROM',

package/src/index.ts

@@ -8,3 +8,4 @@ export * from './l1_metrics.js';
 export * from './wrappers/index.js';
 export * from './start.js';
 export * from './otel_propagation.js';
+export * from './json_rpc_server_metrics.js';

package/src/json_rpc_server_metrics.ts

@@ -0,0 +1,151 @@
+import { Timer } from '@aztec/foundation/timer';
+
+import type Koa from 'koa';
+
+import * as Attributes from './attributes.js';
+import * as Metrics from './metrics.js';
+import { getTelemetryClient } from './start.js';
+import type { Histogram, MetricAttributesType, TelemetryClient, UpDownCounter } from './telemetry.js';
+import { ATTR_JSONRPC_METHOD, ATTR_JSONRPC_SERVICE } from './vendor/attributes.js';
+
+const BATCH_SIZE_BUCKETS = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000];
+
+/** Fixed reasons for rejecting an RPC request before dispatching it to a registered handler. */
+export type JsonRpcRejectionReason =
+  | 'unauthorized'
+  | 'parse_error'
+  | 'invalid_request'
+  | 'method_not_found'
+  | 'bad_request'
+  | 'internal_error';
+
+/** Records bounded-cardinality metrics for registered JSON-RPC calls, rejected requests, and batches. */
+export class JsonRpcServerMetrics {
+  private readonly requestCount: UpDownCounter;
+  private readonly requestDuration: Histogram;
+  private readonly requestValidationDuration: Histogram;
+  private readonly rejectedRequestCount: UpDownCounter;
+  private readonly batchCount: UpDownCounter;
+  private readonly batchDuration: Histogram;
+  private readonly batchSize: Histogram;
+
+  constructor(telemetry: TelemetryClient) {
+    const meter = telemetry.getMeter('JsonRpcServer');
+    this.requestCount = meter.createUpDownCounter(Metrics.JSON_RPC_SERVER_REQUEST_COUNT);
+    this.requestDuration = meter.createHistogram(Metrics.JSON_RPC_SERVER_REQUEST_DURATION);
+    this.requestValidationDuration = meter.createHistogram(Metrics.JSON_RPC_SERVER_REQUEST_VALIDATION_DURATION);
+    this.rejectedRequestCount = meter.createUpDownCounter(Metrics.JSON_RPC_SERVER_REJECTED_REQUEST_COUNT);
+    this.batchCount = meter.createUpDownCounter(Metrics.JSON_RPC_SERVER_BATCH_COUNT);
+    this.batchDuration = meter.createHistogram(Metrics.JSON_RPC_SERVER_BATCH_DURATION);
+    this.batchSize = meter.createHistogram(Metrics.JSON_RPC_SERVER_BATCH_SIZE, {
+      advice: { explicitBucketBoundaries: BATCH_SIZE_BUCKETS },
+    });
+  }
+
+  /** Records the outcome and handler duration of a registered RPC method. */
+  public recordRequest(fullMethod: string, durationMs: number, ok: boolean): void {
+    const [service, method] = splitJsonRpcMethod(fullMethod);
+    const attributes: MetricAttributesType = {
+      ...(service === undefined ? {} : { [ATTR_JSONRPC_SERVICE]: service }),
+      [ATTR_JSONRPC_METHOD]: method,
+      [Attributes.OK]: ok,
+    };
+    this.requestCount.add(1, attributes);
+    this.requestDuration.record(durationMs, attributes);
+  }
+
+  /** Records a pre-dispatch rejection using a fixed reason. */
+  public recordRejectedRequest(reason: JsonRpcRejectionReason): void {
+    this.rejectedRequestCount.add(1, { [Attributes.JSON_RPC_REJECTION_REASON]: reason });
+  }
+
+  /** Records the duration and outcome of validating a registered RPC method's parameters. */
+  public recordRequestValidation(fullMethod: string, durationMs: number, ok: boolean): void {
+    const [service, method] = splitJsonRpcMethod(fullMethod);
+    this.requestValidationDuration.record(durationMs, {
+      ...(service === undefined ? {} : { [ATTR_JSONRPC_SERVICE]: service }),
+      [ATTR_JSONRPC_METHOD]: method,
+      [Attributes.OK]: ok,
+    });
+  }
+
+  /** Records the outcome, processing duration, and number of calls in a batch envelope. */
+  public recordBatch(size: number, durationMs: number, ok: boolean): void {
+    const attributes = { [Attributes.OK]: ok };
+    this.batchCount.add(1, attributes);
+    this.batchDuration.record(durationMs, attributes);
+    this.batchSize.record(size, attributes);
+  }
+}
+
+let metricsOwner: TelemetryClient | undefined;
+let metrics: JsonRpcServerMetrics | undefined;
+
+export function getJsonRpcServerMetrics(): JsonRpcServerMetrics {
+  const telemetry = getTelemetryClient();
+  if (metricsOwner !== telemetry) {
+    metricsOwner = telemetry;
+    metrics = new JsonRpcServerMetrics(telemetry);
+  }
+  return metrics!;
+}
+
+export function getOtelJsonRpcServerMetricsMiddleware(
+  metricsProvider: () => Pick<JsonRpcServerMetrics, 'recordBatch' | 'recordRejectedRequest'> = getJsonRpcServerMetrics,
+): (ctx: Koa.Context, next: () => Promise<void>) => Promise<void> {
+  return async function otelJsonRpcServerMetrics(ctx, next) {
+    const timer = new Timer();
+    await next();
+
+    const requestBody = (ctx.request as { body?: unknown }).body;
+    if (Array.isArray(requestBody)) {
+      metricsProvider().recordBatch(requestBody.length, timer.ms(), Array.isArray(ctx.body));
+    }
+
+    for (const reason of getRejectionReasons(ctx.status, ctx.body)) {
+      metricsProvider().recordRejectedRequest(reason);
+    }
+  };
+}
+
+export function splitJsonRpcMethod(fullMethod: string): [service: string | undefined, method: string] {
+  const separator = fullMethod.indexOf('_');
+  return separator === -1 ? [undefined, fullMethod] : [fullMethod.slice(0, separator), fullMethod.slice(separator + 1)];
+}
+
+function getRejectionReasons(status: number, response: unknown): JsonRpcRejectionReason[] {
+  if (status === 401) {
+    return ['unauthorized'];
+  }
+
+  const responses = Array.isArray(response) ? response : [response];
+  return responses.flatMap(item => {
+    const code = getErrorCode(item);
+    if (code === -32700) {
+      return ['parse_error'];
+    }
+    if (code === -32601) {
+      return ['method_not_found'];
+    }
+    if (code === -32600) {
+      return ['invalid_request'];
+    }
+    if (code === -32603) {
+      return ['internal_error'];
+    }
+    if (code === -32000) {
+      return ['bad_request'];
+    }
+    return [];
+  });
+}
+
+function getErrorCode(response: unknown): number | undefined {
+  if (!response || typeof response !== 'object' || !('error' in response)) {
+    return undefined;
+  }
+  const error = response.error;
+  return error && typeof error === 'object' && 'code' in error && typeof error.code === 'number'
+    ? error.code
+    : undefined;
+}

package/src/metrics.ts

@@ -33,6 +33,46 @@ export interface MetricDefinition {
   readonly valueType?: ValueType;
 }
 
+export const JSON_RPC_SERVER_REQUEST_COUNT: MetricDefinition = {
+  name: 'aztec.json_rpc.server.request_count',
+  description: 'Number of completed JSON-RPC server requests',
+  valueType: ValueType.INT,
+};
+export const JSON_RPC_SERVER_REQUEST_DURATION: MetricDefinition = {
+  name: 'aztec.json_rpc.server.request_duration',
+  description: 'JSON-RPC server request handler duration',
+  unit: 'ms',
+  valueType: ValueType.DOUBLE,
+};
+export const JSON_RPC_SERVER_REQUEST_VALIDATION_DURATION: MetricDefinition = {
+  name: 'aztec.json_rpc.server.request_validation_duration',
+  description: 'JSON-RPC server request parameter validation duration',
+  unit: 'ms',
+  valueType: ValueType.DOUBLE,
+};
+export const JSON_RPC_SERVER_REJECTED_REQUEST_COUNT: MetricDefinition = {
+  name: 'aztec.json_rpc.server.rejected_request_count',
+  description: 'Number of JSON-RPC requests rejected before handler dispatch',
+  valueType: ValueType.INT,
+};
+export const JSON_RPC_SERVER_BATCH_COUNT: MetricDefinition = {
+  name: 'aztec.json_rpc.server.batch_count',
+  description: 'Number of JSON-RPC batches received',
+  valueType: ValueType.INT,
+};
+export const JSON_RPC_SERVER_BATCH_DURATION: MetricDefinition = {
+  name: 'aztec.json_rpc.server.batch_duration',
+  description: 'JSON-RPC batch processing duration',
+  unit: 'ms',
+  valueType: ValueType.DOUBLE,
+};
+export const JSON_RPC_SERVER_BATCH_SIZE: MetricDefinition = {
+  name: 'aztec.json_rpc.server.batch_size',
+  description: 'Number of requests in a JSON-RPC batch',
+  unit: 'requests',
+  valueType: ValueType.INT,
+};
+
 export const BLOB_SINK_STORE_REQUESTS: MetricDefinition = {
   name: 'aztec.blob_sink.store_request_count',
   description: 'Number of blob store requests',
@@ -162,7 +202,7 @@ export const MEMPOOL_TX_ADDED_COUNT: MetricDefinition = {
 };
 export const MEMPOOL_TX_MINED_DELAY: MetricDefinition = {
   name: 'aztec.mempool.tx_mined_delay',
-  description: 'Delay between transaction added and evicted from the mempool',
+  description: 'Delay (ms) from a transaction being received into the pool to being mined',
   unit: 'ms',
   valueType: ValueType.INT,
 };
@@ -204,6 +244,24 @@ export const MEMPOOL_TX_POOL_V2_METADATA_MEMORY: MetricDefinition = {
   valueType: ValueType.INT,
 };
 
+export const MEMPOOL_TX_POOL_V2_QUEUE_WAIT: MetricDefinition = {
+  name: 'aztec.mempool.tx_pool_v2.queue_wait',
+  description: 'Time an operation waits in the tx pool serial queue before executing, keyed by mempool.operation',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const MEMPOOL_TX_POOL_V2_QUEUE_EXECUTION: MetricDefinition = {
+  name: 'aztec.mempool.tx_pool_v2.queue_execution',
+  description: 'Time an operation spends executing in the tx pool serial queue, keyed by mempool.operation',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const MEMPOOL_TX_POOL_V2_QUEUE_LENGTH: MetricDefinition = {
+  name: 'aztec.mempool.tx_pool_v2.queue_length',
+  description: 'Number of operations waiting in the tx pool serial queue',
+  valueType: ValueType.INT,
+};
+
 export const MEMPOOL_TX_POOL_V2_DUPLICATE_ADD: MetricDefinition = {
   name: 'aztec.mempool.tx_pool_v2.duplicate_add',
   description: 'Transactions received via addPendingTxs that were already in the pool',
@@ -308,6 +366,12 @@ export const ARCHIVER_SYNC_PER_BLOCK: MetricDefinition = {
   unit: 'ms',
   valueType: ValueType.INT,
 };
+export const ARCHIVER_SYNC_PER_CHECKPOINT: MetricDefinition = {
+  name: 'aztec.archiver.checkpoint.sync_per_item_duration',
+  description: 'Duration to sync a checkpoint',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
 export const ARCHIVER_SYNC_BLOCK_COUNT: MetricDefinition = {
   name: 'aztec.archiver.block.sync_count',
   description: 'Number of blocks synced from L1',
@@ -334,7 +398,8 @@ export const ARCHIVER_PRUNE_DURATION: MetricDefinition = {
 };
 export const ARCHIVER_PRUNE_COUNT: MetricDefinition = {
   name: 'aztec.archiver.prune_count',
-  description: 'Number of prunes detected',
+  description:
+    'Number of prunes detected, dimensioned by prune_type: unproven (epoch prune of checkpoints that will not be proven), uncheckpointed (proposed blocks whose slot ended without a checkpoint), l1_conflict (proposed blocks conflicting with an L1 checkpoint), orphan (proposed blocks whose matching proposed checkpoint never arrived before the deadline), and l1_mismatch (the local checkpointed tip diverged from L1 — an L1 reorg or a pruned/missed-proof checkpoint — rewinding already-checkpointed blocks).',
   valueType: ValueType.INT,
 };
 
@@ -357,6 +422,12 @@ export const ARCHIVER_CHECKPOINT_L1_INCLUSION_DELAY: MetricDefinition = {
   valueType: ValueType.INT,
 };
 
+export const ARCHIVER_CHECKPOINT_PROMOTED_COUNT: MetricDefinition = {
+  name: 'aztec.archiver.checkpoint_promoted_count',
+  description: 'Number of checkpoints promoted from proposed (blob fetch skipped)',
+  valueType: ValueType.INT,
+};
+
 export const NODE_RECEIVE_TX_DURATION: MetricDefinition = {
   name: 'aztec.node.receive_tx.duration',
   description: 'The duration of the receiveTx method',
@@ -388,6 +459,12 @@ export const SEQUENCER_STATE_TRANSITION_BUFFER_DURATION: MetricDefinition = {
   unit: 'ms',
   valueType: ValueType.INT,
 };
+export const SEQUENCER_STATE_DURATION: MetricDefinition = {
+  name: 'aztec.sequencer.state_duration',
+  description: 'Wall-clock time spent in each sequencer state, labelled by the state being left',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
 export const SEQUENCER_BLOCK_BUILD_DURATION: MetricDefinition = {
   name: 'aztec.sequencer.block.build_duration',
   description: 'Duration to build a block',
@@ -405,6 +482,12 @@ export const SEQUENCER_BLOCK_COUNT: MetricDefinition = {
   description: 'Number of blocks built by this sequencer',
   valueType: ValueType.INT,
 };
+export const SEQUENCER_BLOCK_INTER_BLOCK_TIME: MetricDefinition = {
+  name: 'aztec.sequencer.block.inter_block_time',
+  description: 'Wall-clock time elapsed between consecutive blocks being built by this sequencer',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
 export const SEQUENCER_CURRENT_SLOT_REWARDS: MetricDefinition = {
   name: 'aztec.sequencer.current_slot_rewards',
   description: 'The rewards earned per filled slot',
@@ -477,6 +560,25 @@ export const SEQUENCER_CHECKPOINT_BUILD_DURATION: MetricDefinition = {
   unit: 'ms',
   valueType: ValueType.INT,
 };
+export const SEQUENCER_CHECKPOINT_START_TO_FIRST_BLOCK_DURATION: MetricDefinition = {
+  name: 'aztec.sequencer.checkpoint.start_to_first_block_duration',
+  description: 'Time from starting checkpoint work to the first block finishing build',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const SEQUENCER_CHECKPOINT_LAST_BLOCK_TO_BROADCAST_DURATION: MetricDefinition = {
+  name: 'aztec.sequencer.checkpoint.last_block_to_broadcast_duration',
+  description: 'Time from the final block finishing build to the checkpoint proposal being broadcast',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const SEQUENCER_PIPELINED_CHECKPOINT_BUILD_START_OFFSET_FROM_SLOT_BOUNDARY: MetricDefinition = {
+  name: 'aztec.sequencer.pipelined_checkpoint.build_start_offset_from_slot_boundary',
+  description:
+    'Absolute offset from the wall-clock slot boundary when a pipelined checkpoint build starts. Use aztec.slot_boundary_side to distinguish before vs after the boundary.',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
 export const SEQUENCER_CHECKPOINT_BLOCK_COUNT: MetricDefinition = {
   name: 'aztec.sequencer.checkpoint.block_count',
   description: 'Number of blocks built in a checkpoint',
@@ -499,6 +601,46 @@ export const SEQUENCER_SLASHING_ATTEMPTS_COUNT: MetricDefinition = {
   description: 'The number of slashing action attempts',
   valueType: ValueType.INT,
 };
+export const SLASHER_ROUND_EXECUTED_COUNT: MetricDefinition = {
+  name: 'aztec.slasher.round.executed_count',
+  description: 'The number of slashing rounds executed',
+  valueType: ValueType.INT,
+};
+export const SLASHER_OWN_VALIDATOR_TARGETED_COUNT: MetricDefinition = {
+  name: 'aztec.slasher.own_validator.targeted_count',
+  description:
+    "The number of times one of the node's own validators was named as a slash target by an onchain vote, counted " +
+    'once per vote per validator. Votes cast while the node is offline or starting up are not counted, and L1 ' +
+    'reorgs can cause small drift within a round.',
+  valueType: ValueType.INT,
+};
+export const SLASHER_OWN_VALIDATOR_CURRENT_ROUND_VOTES_MAX: MetricDefinition = {
+  name: 'aztec.slasher.own_validator.current_round_votes_max',
+  description:
+    "Highest number of votes cast against any committee position held by the node's own validators in the current " +
+    'slashing round, to compare against aztec.slasher.quorum_size (the contract tallies quorum per position). ' +
+    'Resets to zero when a new round starts. Votes cast while the node is offline or starting up are not counted, ' +
+    'and L1 reorgs can cause small drift within a round.',
+  valueType: ValueType.INT,
+};
+export const SLASHER_QUORUM_SIZE: MetricDefinition = {
+  name: 'aztec.slasher.quorum_size',
+  description: 'The number of votes a validator must receive in a round to be slashed',
+  valueType: ValueType.INT,
+};
+export const SLASHER_OWN_VALIDATOR_SLASHED_COUNT: MetricDefinition = {
+  name: 'aztec.slasher.own_validator.slashed_count',
+  description: "The number of times one of the node's own validators was slashed, from executed Slashed events",
+  valueType: ValueType.INT,
+};
+export const SLASHER_OWN_VALIDATOR_SLASHED_AMOUNT: MetricDefinition = {
+  name: 'aztec.slasher.own_validator.slashed_amount',
+  description:
+    "Cumulative amount slashed from the node's own validators in whole staking-asset tokens, from executed " +
+    'Slashed events',
+  unit: 'tokens',
+  valueType: ValueType.DOUBLE,
+};
 export const SEQUENCER_CHECKPOINT_SUCCESS_COUNT: MetricDefinition = {
   name: 'aztec.sequencer.checkpoint.success_count',
   description: 'The number of times checkpoint publishing succeeded',
@@ -514,6 +656,11 @@ export const SEQUENCER_PIPELINE_DISCARDS_COUNT: MetricDefinition = {
   description: 'The number of times a pipeline was discarded',
   valueType: ValueType.INT,
 };
+export const SEQUENCER_PIPELINE_PARENT_CHECKPOINT_MISMATCH_COUNT: MetricDefinition = {
+  name: 'aztec.sequencer.pipeline.parent_checkpoint_mismatch_count',
+  description: 'The number of times a pipelined checkpoint was discarded because the parent did not match expectations',
+  valueType: ValueType.INT,
+};
 
 // Fisherman fee analysis metrics
 export const FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED: MetricDefinition = {
@@ -681,6 +828,24 @@ export const L1_PUBLISHER_TX_TOTAL_FEE: MetricDefinition = {
   unit: 'eth',
   valueType: ValueType.DOUBLE,
 };
+export const PROVER_NODE_ESTIMATED_SUBMISSION_GAS: MetricDefinition = {
+  name: 'aztec.prover_node.estimated_submission.gas',
+  description: 'Estimated gas for a proof submission tx (proof publishing disabled, not actually sent)',
+  unit: 'gas',
+  valueType: ValueType.INT,
+};
+export const PROVER_NODE_ESTIMATED_SUBMISSION_GAS_PRICE: MetricDefinition = {
+  name: 'aztec.prover_node.estimated_submission.gas_price',
+  description: 'Estimated effective gas price for a proof submission tx (proof publishing disabled, not actually sent)',
+  unit: 'gwei',
+  valueType: ValueType.DOUBLE,
+};
+export const PROVER_NODE_ESTIMATED_SUBMISSION_TOTAL_FEE: MetricDefinition = {
+  name: 'aztec.prover_node.estimated_submission.total_fee',
+  description: 'Estimated total L1 fee for a proof submission tx (proof publishing disabled, not actually sent)',
+  unit: 'eth',
+  valueType: ValueType.DOUBLE,
+};
 
 export const L1_BLOCK_HEIGHT: MetricDefinition = {
   name: 'aztec.l1.block_height',
@@ -909,6 +1074,19 @@ export const P2P_GOSSIP_AGG_MESSAGE_VALIDATION_DURATION_AVG: MetricDefinition =
   valueType: ValueType.INT,
 };
 
+export const P2P_GOSSIP_SLOW_VALIDATION_COUNT: MetricDefinition = {
+  name: 'aztec.p2p.gossip.slow_validation_count',
+  description: 'Number of gossip validations that exceeded 75% of the mcache eviction window',
+  valueType: ValueType.INT,
+};
+
+export const P2P_GOSSIP_TX_VALIDATION_STAGE_DURATION: MetricDefinition = {
+  name: 'aztec.p2p.gossip.tx_validation_stage_duration',
+  description: 'Duration of each stage of gossiped tx validation, keyed by the tx_validation_stage attribute',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+
 export const PUBLIC_PROCESSOR_TX_DURATION: MetricDefinition = {
   name: 'aztec.public_processor.tx_duration',
   description: 'Duration to process a public transaction',
@@ -972,6 +1150,17 @@ export const PUBLIC_PROCESSOR_TREE_INSERTION: MetricDefinition = {
   unit: 'ms',
   valueType: ValueType.INT,
 };
+export const PUBLIC_PROCESSOR_SILENTLY_SKIPPED_COUNT: MetricDefinition = {
+  name: 'aztec.public_processor.silently_skipped_count',
+  description: 'Public txs fully processed then skipped (e.g. blob-field limit); not counted as processed or failed',
+  valueType: ValueType.INT,
+};
+export const PUBLIC_PROCESSOR_SILENTLY_SKIPPED_DURATION: MetricDefinition = {
+  name: 'aztec.public_processor.silently_skipped_duration',
+  description: 'Wall-clock time spent processing txs that were then silently skipped',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
 
 export const PUBLIC_EXECUTOR_PREFIX = 'aztec.public_executor.';
 export const PUBLIC_EXECUTOR_SIMULATION_COUNT: MetricDefinition = {
@@ -1116,12 +1305,6 @@ export const PROVING_AGENT_IDLE: MetricDefinition = {
   valueType: ValueType.DOUBLE,
 };
 
-export const PROVER_NODE_EXECUTION_DURATION: MetricDefinition = {
-  name: 'aztec.prover_node.execution.duration',
-  description: 'Duration of execution of an epoch by the prover',
-  unit: 'ms',
-  valueType: ValueType.INT,
-};
 export const PROVER_NODE_JOB_DURATION: MetricDefinition = {
   name: 'aztec.prover_node.job_duration',
   description: 'Duration of proving job',
@@ -1149,12 +1332,6 @@ export const PROVER_NODE_BLOB_PROCESSING_LAST_DURATION: MetricDefinition = {
   unit: 'ms',
   valueType: ValueType.INT,
 };
-export const PROVER_NODE_CHONK_VERIFIER_LAST_DURATION: MetricDefinition = {
-  name: 'aztec.prover_node.chonk_verifier.last_duration',
-  description: 'Duration of chonk verifier enqueuing in epoch proving job',
-  unit: 'ms',
-  valueType: ValueType.INT,
-};
 export const PROVER_NODE_BLOCK_PROCESSING_DURATION: MetricDefinition = {
   name: 'aztec.prover_node.block_processing.duration',
   description: 'Duration of processing a single block in epoch proving job',
@@ -1167,12 +1344,33 @@ export const PROVER_NODE_CHECKPOINT_PROCESSING_DURATION: MetricDefinition = {
   unit: 'ms',
   valueType: ValueType.INT,
 };
-export const PROVER_NODE_ALL_CHECKPOINTS_PROCESSING_LAST_DURATION: MetricDefinition = {
-  name: 'aztec.prover_node.all_checkpoints_processing.last_duration',
-  description: 'Duration of processing all checkpoints in epoch proving job',
+export const PROVER_NODE_CHECKPOINT_BLOCKS: MetricDefinition = {
+  name: 'aztec.prover_node.checkpoint_blocks',
+  description: 'Number of blocks in a proven checkpoint',
+  valueType: ValueType.INT,
+};
+export const PROVER_NODE_CHECKPOINT_TRANSACTIONS: MetricDefinition = {
+  name: 'aztec.prover_node.checkpoint_transactions',
+  description: 'Number of transactions in a proven checkpoint',
+  valueType: ValueType.INT,
+};
+export const PROVER_NODE_CHECKPOINT_PROVING_DURATION: MetricDefinition = {
+  name: 'aztec.prover_node.checkpoint_proving.duration',
+  description:
+    'Duration from the start of checkpoint processing to its block proofs being ready (excludes tx gathering)',
   unit: 'ms',
   valueType: ValueType.INT,
 };
+export const PROVER_NODE_ACTIVE_CHECKPOINTS: MetricDefinition = {
+  name: 'aztec.prover_node.active_checkpoints',
+  description: 'Current number of canonical CheckpointProvers in the store (i.e. checkpoints currently being proven)',
+  valueType: ValueType.INT,
+};
+export const PROVER_NODE_ACTIVE_EPOCH_SESSIONS: MetricDefinition = {
+  name: 'aztec.prover_node.active_epoch_sessions',
+  description: 'Current number of live EpochSessions, broken down by kind (full|partial)',
+  valueType: ValueType.INT,
+};
 export const PROVER_NODE_REWARDS_TOTAL: MetricDefinition = {
   name: 'aztec.prover_node.rewards_total',
   description: 'The rewards earned (total)',
@@ -1286,6 +1484,19 @@ export const VALIDATOR_RE_EXECUTION_TX_COUNT: MetricDefinition = {
   unit: 'tx',
   valueType: ValueType.INT,
 };
+export const VALIDATOR_CHECKPOINT_PROPOSAL_TO_PIPELINED_STATE_DURATION: MetricDefinition = {
+  name: 'aztec.validator.checkpoint_proposal_to_pipelined_state_duration',
+  description: 'Time from receiving a checkpoint proposal to setting proposed checkpoint state for pipelining',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
+export const VALIDATOR_CHECKPOINT_PROPOSAL_RECEIVE_OFFSET_FROM_NEXT_SLOT_BOUNDARY: MetricDefinition = {
+  name: 'aztec.validator.checkpoint.proposal_receive_offset_from_next_slot_boundary',
+  description:
+    'Absolute offset from the next slot boundary when a foreign checkpoint proposal is received. Use aztec.slot_boundary_side to distinguish before vs after the boundary.',
+  unit: 'ms',
+  valueType: ValueType.INT,
+};
 
 export const VALIDATOR_FAILED_REEXECUTION_COUNT: MetricDefinition = {
   name: 'aztec.validator.failed_reexecution_count',