@aztec/telemetry-client 0.0.0-test.1 → 0.0.1-commit.5476d83

package/src/{event_loop_monitor.ts → nodejs_metrics_monitor.ts}

@@ -1,3 +1,4 @@
+import type { Observable } from '@opentelemetry/api';
 import { type EventLoopUtilization, type IntervalHistogram, monitorEventLoopDelay, performance } from 'node:perf_hooks';
 
 import * as Attributes from './attributes.js';
@@ -13,7 +14,7 @@ import {
 /**
  * Detector for custom Aztec attributes
  */
-export class EventLoopMonitor {
+export class NodejsMetricsMonitor {
   private eventLoopDelayGauges: {
     min: ObservableGauge;
     max: ObservableGauge;
@@ -24,6 +25,10 @@ export class EventLoopMonitor {
     p99: ObservableGauge;
   };
 
+  // skip `rss` because that's already tracked by @opentelemetry/host-metrics
+  // description of each field here https://nodejs.org/api/process.html#processmemoryusage
+  private memoryGauges: Record<Exclude<keyof NodeJS.MemoryUsage, 'rss'>, ObservableGauge>;
+
   private eventLoopUilization: ObservableGauge;
   private eventLoopTime: UpDownCounter;
 
@@ -62,6 +67,25 @@ export class EventLoopMonitor {
     });
 
     this.eventLoopDelay = monitorEventLoopDelay();
+
+    this.memoryGauges = {
+      heapUsed: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_USAGE, {
+        unit: 'By',
+        description: 'Memory used by the V8 heap',
+      }),
+      heapTotal: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_TOTAL, {
+        unit: 'By',
+        description: 'The max size the V8 heap can grow to',
+      }),
+      arrayBuffers: meter.createObservableGauge(Metrics.NODEJS_MEMORY_BUFFER_USAGE, {
+        unit: 'By',
+        description: 'Memory allocated for buffers (includes native memory used)',
+      }),
+      external: meter.createObservableGauge(Metrics.NODEJS_MEMORY_NATIVE_USAGE, {
+        unit: 'By',
+        description: 'Memory allocated for native C++ objects',
+      }),
+    };
   }
 
   start(): void {
@@ -74,6 +98,7 @@ export class EventLoopMonitor {
     this.meter.addBatchObservableCallback(this.measure, [
       this.eventLoopUilization,
       ...Object.values(this.eventLoopDelayGauges),
+      ...Object.values(this.memoryGauges),
     ]);
   }
 
@@ -84,6 +109,7 @@ export class EventLoopMonitor {
     this.meter.removeBatchObservableCallback(this.measure, [
       this.eventLoopUilization,
       ...Object.values(this.eventLoopDelayGauges),
+      ...Object.values(this.memoryGauges),
     ]);
     this.eventLoopDelay.disable();
     this.eventLoopDelay.reset();
@@ -91,6 +117,20 @@ export class EventLoopMonitor {
   }
 
   private measure = (obs: BatchObservableResult): void => {
+    this.measureMemoryUsage(obs);
+    this.measureEventLoopDelay(obs);
+  };
+
+  private measureMemoryUsage = (observer: BatchObservableResult) => {
+    const mem = process.memoryUsage();
+
+    observer.observe(this.memoryGauges.heapUsed, mem.heapUsed);
+    observer.observe(this.memoryGauges.heapTotal, mem.heapTotal);
+    observer.observe(this.memoryGauges.arrayBuffers, mem.arrayBuffers);
+    observer.observe(this.memoryGauges.external, mem.external);
+  };
+
+  private measureEventLoopDelay = (obs: BatchObservableResult): void => {
     const newELU = performance.eventLoopUtilization();
     const delta = performance.eventLoopUtilization(newELU, this.lastELU);
     this.lastELU = newELU;
@@ -103,17 +143,26 @@ export class EventLoopMonitor {
     // - https://youtu.be/WetXnEPraYM
     obs.observe(this.eventLoopUilization, delta.utilization);
 
-    this.eventLoopTime.add(Math.floor(delta.idle), { [Attributes.NODEJS_EVENT_LOOP_STATE]: 'idle' });
-    this.eventLoopTime.add(Math.floor(delta.active), { [Attributes.NODEJS_EVENT_LOOP_STATE]: 'active' });
+    this.eventLoopTime.add(Math.trunc(delta.idle), { [Attributes.NODEJS_EVENT_LOOP_STATE]: 'idle' });
+    this.eventLoopTime.add(Math.trunc(delta.active), { [Attributes.NODEJS_EVENT_LOOP_STATE]: 'active' });
 
-    obs.observe(this.eventLoopDelayGauges.min, Math.floor(this.eventLoopDelay.min));
-    obs.observe(this.eventLoopDelayGauges.mean, Math.floor(this.eventLoopDelay.mean));
-    obs.observe(this.eventLoopDelayGauges.max, Math.floor(this.eventLoopDelay.max));
-    obs.observe(this.eventLoopDelayGauges.stddev, Math.floor(this.eventLoopDelay.stddev));
-    obs.observe(this.eventLoopDelayGauges.p50, Math.floor(this.eventLoopDelay.percentile(50)));
-    obs.observe(this.eventLoopDelayGauges.p90, Math.floor(this.eventLoopDelay.percentile(90)));
-    obs.observe(this.eventLoopDelayGauges.p99, Math.floor(this.eventLoopDelay.percentile(99)));
+    safeObserveInt(obs, this.eventLoopDelayGauges.min, this.eventLoopDelay.min);
+    safeObserveInt(obs, this.eventLoopDelayGauges.mean, this.eventLoopDelay.mean);
+    safeObserveInt(obs, this.eventLoopDelayGauges.max, this.eventLoopDelay.max);
+    safeObserveInt(obs, this.eventLoopDelayGauges.stddev, this.eventLoopDelay.stddev);
+    safeObserveInt(obs, this.eventLoopDelayGauges.p50, this.eventLoopDelay.percentile(50));
+    safeObserveInt(obs, this.eventLoopDelayGauges.p90, this.eventLoopDelay.percentile(90));
+    safeObserveInt(obs, this.eventLoopDelayGauges.p99, this.eventLoopDelay.percentile(99));
 
     this.eventLoopDelay.reset();
   };
 }
+
+function safeObserveInt(observer: BatchObservableResult, metric: Observable, value: number, attrs?: object) {
+  // discard NaN, Infinity, -Infinity
+  if (!Number.isFinite(value)) {
+    return;
+  }
+
+  observer.observe(metric, Math.trunc(value), attrs);
+}

package/src/noop.ts

@@ -3,6 +3,9 @@ import { type Meter, type Span, type SpanContext, type Tracer, createNoopMeter }
 import type { TelemetryClient } from './telemetry.js';
 
 export class NoopTelemetryClient implements TelemetryClient {
+  setExportedPublicTelemetry(_prefixes: string[]): void {}
+  setPublicTelemetryCollectFrom(_roles: string[]): void {}
+
   getMeter(): Meter {
     return createNoopMeter();
   }
@@ -32,7 +35,7 @@ export class NoopTracer implements Tracer {
     return new NoopSpan();
   }
 
-  startActiveSpan<F extends (...args: any[]) => any>(_name: string, ...args: (unknown | F)[]): ReturnType<F> {
+  startActiveSpan<F extends (...args: any[]) => any>(_name: string, ...args: unknown[]): ReturnType<F> {
     // there are three different signatures for startActiveSpan, grab the function, we don't care about the rest
     const fn = args.find(arg => typeof arg === 'function') as F;
     return fn(new NoopSpan());

package/src/otel.ts

@@ -28,8 +28,8 @@ import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace
 import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
 
 import type { TelemetryClientConfig } from './config.js';
-import { EventLoopMonitor } from './event_loop_monitor.js';
-import { OtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
+import { NodejsMetricsMonitor } from './nodejs_metrics_monitor.js';
+import { OtelFilterMetricExporter, PublicOtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
 import { registerOtelLoggerProvider } from './otel_logger_provider.js';
 import { getOtelResource } from './otel_resource.js';
 import type { TelemetryClient } from './telemetry.js';
@@ -38,7 +38,7 @@ export type OpenTelemetryClientFactory = (resource: IResource, log: Logger) => O
 
 export class OpenTelemetryClient implements TelemetryClient {
   hostMetrics: HostMetrics | undefined;
-  eventLoopMonitor: EventLoopMonitor | undefined;
+  nodejsMetricsMonitor: NodejsMetricsMonitor | undefined;
   private meters: Map<string, Meter> = new Map<string, Meter>();
   private tracers: Map<string, Tracer> = new Map<string, Tracer>();
 
@@ -47,9 +47,18 @@ export class OpenTelemetryClient implements TelemetryClient {
     private meterProvider: MeterProvider,
     private traceProvider: TracerProvider,
     private loggerProvider: LoggerProvider | undefined,
+    private publicMetricExporter: PublicOtelFilterMetricExporter | undefined,
     private log: Logger,
   ) {}
 
+  setExportedPublicTelemetry(metrics: string[]): void {
+    this.publicMetricExporter?.setMetricPrefixes(metrics);
+  }
+
+  setPublicTelemetryCollectFrom(roles: string[]): void {
+    this.publicMetricExporter?.setAllowedRoles(roles);
+  }
+
   getMeter(name: string): Meter {
     let meter = this.meters.get(name);
     if (!meter) {
@@ -91,12 +100,12 @@ export class OpenTelemetryClient implements TelemetryClient {
       meterProvider: this.meterProvider,
     });
 
-    this.eventLoopMonitor = new EventLoopMonitor(
+    this.nodejsMetricsMonitor = new NodejsMetricsMonitor(
       this.meterProvider.getMeter(this.resource.attributes[ATTR_SERVICE_NAME] as string),
     );
 
     this.hostMetrics.start();
-    this.eventLoopMonitor.start();
+    this.nodejsMetricsMonitor.start();
   }
 
   public isEnabled() {
@@ -112,7 +121,7 @@ export class OpenTelemetryClient implements TelemetryClient {
   }
 
   public async stop() {
-    this.eventLoopMonitor?.stop();
+    this.nodejsMetricsMonitor?.stop();
 
     const flushAndShutdown = async (provider?: { forceFlush: () => Promise<void>; shutdown: () => Promise<void> }) => {
       if (!provider) {
@@ -131,16 +140,31 @@ export class OpenTelemetryClient implements TelemetryClient {
 
   public static createMeterProvider(
     resource: IResource,
-    options: Partial<PeriodicExportingMetricReaderOptions>,
+    exporters: Array<PeriodicExportingMetricReaderOptions>,
   ): MeterProvider {
     return new MeterProvider({
       resource,
-      readers: options.exporter
-        ? [new PeriodicExportingMetricReader(options as PeriodicExportingMetricReaderOptions)]
-        : [],
+      readers: exporters.map(options => new PeriodicExportingMetricReader(options)),
 
       views: [
         // Every histogram matching the selector (type + unit) gets these custom buckets assigned
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'Mmana',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            [0.1, 0.5, 1, 2, 4, 8, 10, 25, 50, 100, 500, 1000, 5000, 10000],
+            true,
+          ),
+        }),
+        new View({
+          instrumentType: InstrumentType.HISTOGRAM,
+          instrumentUnit: 'tx',
+          aggregation: new ExplicitBucketHistogramAggregation(
+            // TPS
+            [0.1 * 36, 0.2 * 36, 0.5 * 36, 1 * 36, 2 * 36, 5 * 36, 10 * 36, 15 * 36].map(Math.ceil),
+            true,
+          ),
+        }),
         new View({
           instrumentType: InstrumentType.HISTOGRAM,
           instrumentUnit: 's',
@@ -257,20 +281,55 @@ export class OpenTelemetryClient implements TelemetryClient {
 
       tracerProvider.register();
 
-      const meterProvider = OpenTelemetryClient.createMeterProvider(resource, {
-        exporter: config.metricsCollectorUrl
-          ? new OtelFilterMetricExporter(
-              new OTLPMetricExporter({ url: config.metricsCollectorUrl.href }),
-              config.otelExcludeMetrics ?? [],
-            )
-          : undefined,
-        exportTimeoutMillis: config.otelExportTimeoutMs,
-        exportIntervalMillis: config.otelCollectIntervalMs,
-      });
+      const exporters: PeriodicExportingMetricReaderOptions[] = [];
+      if (config.metricsCollectorUrl) {
+        // Default to a blacklist that is empty (allow all metrics)
+        let filter: string[] = [];
+        let mode: 'allow' | 'deny' = 'deny';
+        if (config.otelExcludeMetrics.length > 0) {
+          // Implement a blacklist as specified in config
+          log.info(`Excluding metrics from export: ${config.otelExcludeMetrics}`);
+          filter = config.otelExcludeMetrics;
+          mode = 'deny';
+        } else if (config.otelIncludeMetrics.length > 0) {
+          // Implement a whitelist as specified in config
+          log.info(`Including only specified metrics for export: ${config.otelIncludeMetrics}`);
+          filter = config.otelIncludeMetrics;
+          mode = 'allow';
+        }
+        exporters.push({
+          exporter: new OtelFilterMetricExporter(
+            new OTLPMetricExporter({ url: config.metricsCollectorUrl.href }),
+            filter,
+            mode,
+          ),
+          exportTimeoutMillis: config.otelExportTimeoutMs,
+          exportIntervalMillis: config.otelCollectIntervalMs,
+        });
+      }
+
+      let publicExporter: PublicOtelFilterMetricExporter | undefined;
+      if (config.publicMetricsCollectorUrl && !config.publicMetricsOptOut) {
+        log.info(`Exporting public metrics: ${config.publicIncludeMetrics}`, {
+          publicMetrics: config.publicIncludeMetrics,
+          collectorUrl: config.publicMetricsCollectorUrl,
+        });
+        publicExporter = new PublicOtelFilterMetricExporter(
+          config.publicMetricsCollectFrom,
+          new OTLPMetricExporter({ url: config.publicMetricsCollectorUrl.href }),
+          config.publicIncludeMetrics,
+        );
+        exporters.push({
+          exporter: publicExporter,
+          exportTimeoutMillis: config.otelExportTimeoutMs,
+          exportIntervalMillis: config.otelCollectIntervalMs,
+        });
+      }
 
+      const meterProvider = OpenTelemetryClient.createMeterProvider(resource, exporters);
       const loggerProvider = registerOtelLoggerProvider(resource, config.logsCollectorUrl);
 
-      return new OpenTelemetryClient(resource, meterProvider, tracerProvider, loggerProvider, log);
+      return new OpenTelemetryClient(resource, meterProvider, tracerProvider, loggerProvider, publicExporter, log);
     };
   }
 

package/src/otel_filter_metric_exporter.ts

@@ -1,8 +1,14 @@
-import type { ExportResult } from '@opentelemetry/core';
+import { type ExportResult, ExportResultCode } from '@opentelemetry/core';
 import type { MetricData, PushMetricExporter, ResourceMetrics } from '@opentelemetry/sdk-metrics';
 
+import { AZTEC_NODE_ROLE } from './attributes.js';
+
 export class OtelFilterMetricExporter implements PushMetricExporter {
-  constructor(private readonly exporter: PushMetricExporter, private readonly excludeMetricPrefixes: string[]) {
+  constructor(
+    private readonly exporter: PushMetricExporter,
+    private metricPrefix: string[],
+    private readonly filter: 'allow' | 'deny' = 'deny',
+  ) {
     if (exporter.selectAggregation) {
       (this as PushMetricExporter).selectAggregation = exporter.selectAggregation.bind(exporter);
     }
@@ -23,9 +29,17 @@ export class OtelFilterMetricExporter implements PushMetricExporter {
   }
 
   private filterMetrics(metrics: MetricData[]): MetricData[] {
-    return metrics.filter(
-      metric => !this.excludeMetricPrefixes.some(prefix => metric.descriptor.name.startsWith(prefix)),
-    );
+    return metrics.filter(metric => {
+      const matched = this.metricPrefix.some(prefix => metric.descriptor.name.startsWith(prefix));
+
+      if (this.filter === 'deny') {
+        return !matched;
+      }
+
+      if (this.filter === 'allow') {
+        return matched;
+      }
+    });
   }
 
   public forceFlush(): Promise<void> {
@@ -35,4 +49,32 @@ export class OtelFilterMetricExporter implements PushMetricExporter {
   public shutdown(): Promise<void> {
     return this.exporter.shutdown();
   }
+
+  public setMetricPrefixes(metrics: string[]) {
+    this.metricPrefix = metrics;
+  }
+}
+
+export class PublicOtelFilterMetricExporter extends OtelFilterMetricExporter {
+  constructor(
+    private allowedRoles: string[],
+    exporter: PushMetricExporter,
+    metricPrefix: string[],
+  ) {
+    super(exporter, metricPrefix, 'allow');
+  }
+
+  public override export(metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void): void {
+    const role = String(metrics.resource.attributes[AZTEC_NODE_ROLE] ?? '');
+    if (!role || !this.allowedRoles.includes(role)) {
+      // noop
+      return resultCallback({ code: ExportResultCode.SUCCESS });
+    }
+
+    super.export(metrics, resultCallback);
+  }
+
+  public setAllowedRoles(roles: string[]) {
+    this.allowedRoles = roles;
+  }
 }

package/src/otel_resource.ts

@@ -1,16 +1,54 @@
 import {
+  type DetectorSync,
   type IResource,
+  Resource,
   detectResourcesSync,
   envDetectorSync,
   osDetectorSync,
-  processDetectorSync,
   serviceInstanceIdDetectorSync,
 } from '@opentelemetry/resources';
+import { SEMRESATTRS_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
+
+import { AZTEC_NODE_ROLE, AZTEC_REGISTRY_ADDRESS, AZTEC_ROLLUP_ADDRESS, AZTEC_ROLLUP_VERSION } from './attributes.js';
 
 export function getOtelResource(): IResource {
   const resource = detectResourcesSync({
-    detectors: [osDetectorSync, envDetectorSync, processDetectorSync, serviceInstanceIdDetectorSync],
+    detectors: [
+      aztecNetworkDetectorSync,
+      osDetectorSync,
+      envDetectorSync,
+      // this detector is disabled because:
+      // 1. our software runs in a docker container, a lot of the attributes detected would be identical across different machines (e.g. all run node v22, executing the same script, running PID 1, etc)
+      // 2. it catures process.argv which could contain sensitive values in plain text (e.g. validator private keys)
+      // processDetectorSync,
+      serviceInstanceIdDetectorSync,
+    ],
   });
 
   return resource;
 }
+
+const aztecNetworkDetectorSync: DetectorSync = {
+  detect(): IResource {
+    let role: string | undefined;
+    if (process.argv.includes('--sequencer')) {
+      role = 'sequencer';
+    } else if (process.argv.includes('--prover-node')) {
+      role = 'prover-node';
+    } else if (process.argv.includes('--node')) {
+      role = 'node';
+    } else if (process.argv.includes('--p2p-bootstrap')) {
+      role = 'bootnode';
+    }
+    const aztecAttributes = {
+      // this gets overwritten by OTEL_RESOURCE_ATTRIBUTES (if set)
+      [SEMRESATTRS_SERVICE_NAME]: role ? `aztec-${role}` : undefined,
+      [AZTEC_NODE_ROLE]: role,
+      [AZTEC_ROLLUP_VERSION]: process.env.ROLLUP_VERSION ?? 'canonical',
+      [AZTEC_ROLLUP_ADDRESS]: process.env.ROLLUP_CONTRACT_ADDRESS,
+      [AZTEC_REGISTRY_ADDRESS]: process.env.REGISTRY_CONTRACT_ADDRESS,
+    };
+
+    return new Resource(aztecAttributes);
+  },
+};