@aztec/telemetry-client 0.0.0-test.0 → 0.0.1-commit.023c3e5

package/src/bench.ts

@@ -1,12 +1,14 @@
+import { createLogger } from '@aztec/foundation/log';
+
 import type { BatchObservableCallback, Context, MetricOptions, Observable, ValueType } from '@opentelemetry/api';
 
+import type { MetricDefinition } from './metrics.js';
 import { NoopTracer } from './noop.js';
 import type {
   AttributesType,
   Gauge,
   Histogram,
   Meter,
-  MetricsType,
   ObservableGauge,
   ObservableUpDownCounter,
   TelemetryClient,
@@ -29,6 +31,14 @@ export type BenchmarkMetricsType = {
 export class BenchmarkTelemetryClient implements TelemetryClient {
   private meters: InMemoryPlainMeter[] = [];
 
+  constructor() {
+    const log = createLogger('telemetry:client');
+    log.info(`Using benchmark telemetry client`);
+  }
+
+  setExportedPublicTelemetry(_prefixes: string[]): void {}
+  setPublicTelemetryCollectFrom(_roles: string[]): void {}
+
   getMeter(name: string): Meter {
     const meter = new InMemoryPlainMeter(name);
     this.meters.push(meter);
@@ -51,6 +61,14 @@ export class BenchmarkTelemetryClient implements TelemetryClient {
     return true;
   }
 
+  getTraceContext(): string | undefined {
+    return undefined;
+  }
+
+  extractPropagatedContext() {
+    return undefined;
+  }
+
   getMeters(): BenchmarkMetricsType {
     return this.meters;
   }
@@ -69,30 +87,34 @@ class InMemoryPlainMeter implements Meter {
     this.metrics.forEach(metric => metric.clear());
   }
 
-  createGauge(name: MetricsType, options?: MetricOptions | undefined): Gauge {
-    return this.createMetric('gauge', name, options);
+  createGauge(metric: MetricDefinition): Gauge {
+    return this.createMetric('gauge', metric);
   }
 
-  createObservableGauge(name: MetricsType, options?: MetricOptions | undefined): ObservableGauge {
-    return this.createMetric('gauge', name, options);
+  createObservableGauge(metric: MetricDefinition): ObservableGauge {
+    return this.createMetric('gauge', metric);
   }
 
-  createHistogram(name: MetricsType, options?: MetricOptions | undefined): Histogram {
-    return this.createMetric('histogram', name, options);
+  createHistogram(metric: MetricDefinition, _extraOptions?: Partial<MetricOptions>): Histogram {
+    return this.createMetric('histogram', metric);
   }
 
-  createUpDownCounter(name: MetricsType, options?: MetricOptions | undefined): UpDownCounter {
-    return this.createMetric('counter', name, options);
+  createUpDownCounter(metric: MetricDefinition): UpDownCounter {
+    return this.createMetric('counter', metric);
   }
 
-  createObservableUpDownCounter(name: MetricsType, options?: MetricOptions | undefined): ObservableUpDownCounter {
-    return this.createMetric('counter', name, options);
+  createObservableUpDownCounter(metric: MetricDefinition): ObservableUpDownCounter {
+    return this.createMetric('counter', metric);
   }
 
-  private createMetric(type: 'gauge' | 'counter' | 'histogram', name: string, options?: MetricOptions) {
-    const metric = new InMemoryPlainMetric(type, name, options);
-    this.metrics.push(metric);
-    return metric;
+  private createMetric(type: 'gauge' | 'counter' | 'histogram', metric: MetricDefinition) {
+    const m = new InMemoryPlainMetric(type, metric.name, {
+      description: metric.description,
+      unit: metric.unit,
+      valueType: metric.valueType,
+    });
+    this.metrics.push(m);
+    return m;
   }
 
   addBatchObservableCallback(

package/src/config.ts

@@ -1,12 +1,17 @@
-import { type ConfigMappingsType, getConfigFromMappings } from '@aztec/foundation/config';
+import { type ConfigMappingsType, booleanConfigHelper, getConfigFromMappings } from '@aztec/foundation/config';
 
 export interface TelemetryClientConfig {
   metricsCollectorUrl?: URL;
+  publicMetricsCollectorUrl?: URL;
+  publicIncludeMetrics: string[];
+  publicMetricsOptOut: boolean;
+  publicMetricsCollectFrom: string[];
   tracesCollectorUrl?: URL;
   logsCollectorUrl?: URL;
   otelCollectIntervalMs: number;
   otelExportTimeoutMs: number;
-  otelExcludeMetrics?: string[];
+  otelExcludeMetrics: string[];
+  otelIncludeMetrics: string[];
 }
 
 export const telemetryClientConfigMappings: ConfigMappingsType<TelemetryClientConfig> = {
@@ -49,6 +54,53 @@ export const telemetryClientConfigMappings: ConfigMappingsType<TelemetryClientCo
         : [],
     defaultValue: [],
   },
+  otelIncludeMetrics: {
+    env: 'OTEL_INCLUDE_METRICS',
+    description: 'A list of metric prefixes to include in export (ignored if OTEL_EXCLUDE_METRICS is set)',
+    parseEnv: (val: string) =>
+      val
+        ? val
+            .split(',')
+            .map(s => s.trim())
+            .filter(s => s.length > 0)
+        : [],
+    defaultValue: [],
+  },
+
+  publicMetricsCollectorUrl: {
+    env: 'PUBLIC_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT',
+    description: 'A URL to publish a subset of metrics for public consumption',
+    parseEnv: (val: string) => val && new URL(val),
+  },
+  publicMetricsCollectFrom: {
+    env: 'PUBLIC_OTEL_COLLECT_FROM',
+    description: 'The role types to collect metrics from',
+    parseEnv: (val: string) =>
+      val
+        ? val
+            .split(',')
+            .map(s => s.trim())
+            .filter(s => s.length > 0)
+        : [],
+    defaultValue: [],
+  },
+  publicIncludeMetrics: {
+    env: 'PUBLIC_OTEL_INCLUDE_METRICS',
+    description: 'A list of metric prefixes to publicly export',
+    parseEnv: (val: string) =>
+      val
+        ? val
+            .split(',')
+            .map(s => s.trim())
+            .filter(s => s.length > 0)
+        : [],
+    defaultValue: [],
+  },
+  publicMetricsOptOut: {
+    env: 'PUBLIC_OTEL_OPT_OUT',
+    description: 'Whether to opt out of sharing optional telemetry',
+    ...booleanConfigHelper(true),
+  },
 };
 
 export function getConfigEnvVars(): TelemetryClientConfig {

package/src/index.ts

@@ -4,6 +4,7 @@ export * as Attributes from './attributes.js';
 export * from './with_tracer.js';
 export * from './prom_otel_adapter.js';
 export * from './lmdb_metrics.js';
+export * from './l1_metrics.js';
 export * from './wrappers/index.js';
 export * from './start.js';
 export * from './otel_propagation.js';

package/src/l1_metrics.ts

@@ -0,0 +1,65 @@
+import type { EthAddress } from '@aztec/foundation/eth-address';
+
+import { type Chain, type FallbackTransport, type Hex, type HttpTransport, type PublicClient, formatEther } from 'viem';
+
+import { L1_SENDER } from './attributes.js';
+import { L1_BALANCE_ETH, L1_BLOB_BASE_FEE_WEI, L1_BLOCK_HEIGHT, L1_GAS_PRICE_WEI } from './metrics.js';
+import type { BatchObservableResult, Meter, ObservableGauge } from './telemetry.js';
+
+export class L1Metrics {
+  private l1BlockHeight: ObservableGauge;
+  private l1BalanceEth: ObservableGauge;
+  private gasPriceWei: ObservableGauge;
+  private blobBaseFeeWei: ObservableGauge;
+  private addresses: Hex[];
+
+  constructor(
+    private meter: Meter,
+    private client: PublicClient<FallbackTransport<HttpTransport[]>, Chain>,
+    addresses: EthAddress[],
+  ) {
+    this.l1BlockHeight = meter.createObservableGauge(L1_BLOCK_HEIGHT);
+    this.l1BalanceEth = meter.createObservableGauge(L1_BALANCE_ETH);
+    this.gasPriceWei = meter.createObservableGauge(L1_GAS_PRICE_WEI);
+    this.blobBaseFeeWei = meter.createObservableGauge(L1_BLOB_BASE_FEE_WEI);
+
+    this.addresses = addresses.map(addr => addr.toString());
+  }
+
+  start() {
+    this.meter.addBatchObservableCallback(this.observe, [this.l1BalanceEth, this.l1BlockHeight]);
+  }
+
+  stop() {
+    this.meter.removeBatchObservableCallback(this.observe, [this.l1BalanceEth, this.l1BlockHeight]);
+  }
+
+  private observe = async (observer: BatchObservableResult) => {
+    const blockNumber = await this.client.getBlockNumber();
+    const ethBalances = await Promise.all(
+      this.addresses.map(address =>
+        this.client.getBalance({
+          address,
+          blockNumber,
+        }),
+      ),
+    );
+
+    const gasPrice = await this.client.getGasPrice();
+    const blobFee = await this.client.getBlobBaseFee();
+
+    observer.observe(this.l1BlockHeight, Number(blockNumber));
+    observer.observe(this.gasPriceWei, Number(gasPrice));
+    observer.observe(this.blobBaseFeeWei, Number(blobFee));
+
+    for (let i = 0; i < ethBalances.length; i++) {
+      const wei = ethBalances[i];
+      const address = this.addresses[i];
+      const eth = parseFloat(formatEther(wei, 'wei'));
+
+      observer.observe(this.l1BalanceEth, eth, {
+        [L1_SENDER]: address,
+      });
+    }
+  };
+}

package/src/lmdb_metrics.ts

@@ -1,36 +1,35 @@
 import * as Metrics from './metrics.js';
-import {
-  type AttributesType,
-  type BatchObservableResult,
-  type Meter,
-  type ObservableGauge,
-  ValueType,
-} from './telemetry.js';
+import type { AttributesType, BatchObservableResult, Meter, ObservableGauge } from './telemetry.js';
 
-export type LmdbStatsCallback = () => Promise<{ mappingSize: number; numItems: number; actualSize: number }>;
+export type LmdbStatsCallback = () => Promise<{
+  mappingSize: number;
+  physicalFileSize: number;
+  numItems: number;
+  actualSize: number;
+}>;
 
 export class LmdbMetrics {
   private dbMapSize: ObservableGauge;
+  private dbPhysicalFileSize: ObservableGauge;
   private dbUsedSize: ObservableGauge;
   private dbNumItems: ObservableGauge;
 
-  constructor(meter: Meter, private attributes?: AttributesType, private getStats?: LmdbStatsCallback) {
-    this.dbMapSize = meter.createObservableGauge(Metrics.DB_MAP_SIZE, {
-      description: 'LMDB Map Size',
-      valueType: ValueType.INT,
-      unit: 'By',
-    });
-    this.dbUsedSize = meter.createObservableGauge(Metrics.DB_USED_SIZE, {
-      description: 'LMDB Used Size',
-      valueType: ValueType.INT,
-      unit: 'By',
-    });
-    this.dbNumItems = meter.createObservableGauge(Metrics.DB_NUM_ITEMS, {
-      description: 'LMDB Num Items',
-      valueType: ValueType.INT,
-    });
+  constructor(
+    meter: Meter,
+    private attributes?: AttributesType,
+    private getStats?: LmdbStatsCallback,
+  ) {
+    this.dbMapSize = meter.createObservableGauge(Metrics.DB_MAP_SIZE);
+    this.dbPhysicalFileSize = meter.createObservableGauge(Metrics.DB_PHYSICAL_FILE_SIZE);
+    this.dbUsedSize = meter.createObservableGauge(Metrics.DB_USED_SIZE);
+    this.dbNumItems = meter.createObservableGauge(Metrics.DB_NUM_ITEMS);
 
-    meter.addBatchObservableCallback(this.recordDBMetrics, [this.dbMapSize, this.dbUsedSize, this.dbNumItems]);
+    meter.addBatchObservableCallback(this.recordDBMetrics, [
+      this.dbMapSize,
+      this.dbPhysicalFileSize,
+      this.dbUsedSize,
+      this.dbNumItems,
+    ]);
   }
 
   private recordDBMetrics = async (observable: BatchObservableResult) => {
@@ -39,6 +38,7 @@ export class LmdbMetrics {
     }
     const metrics = await this.getStats();
     observable.observe(this.dbMapSize, metrics.mappingSize, this.attributes);
+    observable.observe(this.dbPhysicalFileSize, metrics.physicalFileSize, this.attributes);
     observable.observe(this.dbNumItems, metrics.numItems, this.attributes);
     observable.observe(this.dbUsedSize, metrics.actualSize, this.attributes);
   };

package/src/metric-utils.ts

@@ -0,0 +1,75 @@
+import type { AttributeValue, MetricOptions } from '@opentelemetry/api';
+
+import type { MetricDefinition } from './metrics.js';
+import type { AllowedAttributeNames, Meter, MetricAttributesType, UpDownCounter } from './telemetry.js';
+
+/** Extracts OpenTelemetry MetricOptions from a MetricDefinition */
+export function toMetricOptions(def: MetricDefinition): MetricOptions {
+  return {
+    description: def.description,
+    unit: def.unit,
+    valueType: def.valueType,
+  };
+}
+
+/** A mapping of attribute keys to their possible values */
+export type AttributeValuesMap = Partial<Record<AllowedAttributeNames, AttributeValue[]>>;
+
+/**
+ * Expands an attribute values map into all combinations of attribute objects.
+ * Example: { status: ['ok', 'fail'], type: ['a', 'b'] } =>
+ *   [{ status: 'ok', type: 'a' }, { status: 'ok', type: 'b' }, { status: 'fail', type: 'a' }, { status: 'fail', type: 'b' }]
+ */
+export function expandAttributeCombinations(attrMap: AttributeValuesMap): MetricAttributesType[] {
+  const keys = Object.keys(attrMap) as AllowedAttributeNames[];
+  if (keys.length === 0) {
+    return [{}];
+  }
+
+  const result: MetricAttributesType[] = [];
+
+  function generate(index: number, current: MetricAttributesType) {
+    if (index === keys.length) {
+      result.push({ ...current });
+      return;
+    }
+    const key = keys[index];
+    for (const value of attrMap[key]!) {
+      current[key] = value;
+      generate(index + 1, current);
+    }
+  }
+
+  generate(0, {});
+  return result;
+}
+
+/**
+ * Creates an UpDownCounter and initializes it to 0.
+ * @param meter - The meter to create the counter on
+ * @param metric - The metric definition
+ * @param attributes - Optional attributes for initialization. Can be:
+ *   - AttributeValuesMap: mapping of attribute keys to arrays of possible values (generates cartesian product)
+ *   - MetricAttributesType[]: explicit array of attribute objects
+ *   - undefined: initializes without attributes
+ */
+export function createUpDownCounterWithDefault(
+  meter: Meter,
+  metric: MetricDefinition,
+  attributes: AttributeValuesMap | MetricAttributesType[] | undefined = undefined,
+): UpDownCounter {
+  const counter = meter.createUpDownCounter(metric);
+  if (attributes === undefined) {
+    counter.add(0);
+  } else if (Array.isArray(attributes)) {
+    for (const attrs of attributes) {
+      counter.add(0, attrs);
+    }
+  } else {
+    const combinations = expandAttributeCombinations(attributes);
+    for (const attrs of combinations) {
+      counter.add(0, attrs);
+    }
+  }
+  return counter;
+}