@hyperlane-xyz/rebalancer 0.1.0-beta.5a8bd28ab

package/src/metrics/scripts/metrics.ts

@@ -0,0 +1,298 @@
+import { Logger } from 'pino';
+import { Counter, Gauge, Registry } from 'prom-client';
+
+import { ChainName, Token, TokenStandard, WarpCore } from '@hyperlane-xyz/sdk';
+import { Address } from '@hyperlane-xyz/utils';
+
+import {
+  NativeWalletBalance,
+  WarpRouteBalance,
+  XERC20Limit,
+} from '../types.js';
+
+export const metricsRegister = new Registry();
+
+type SupportedTokenStandards = TokenStandard | 'EvmManagedLockbox' | 'xERC20';
+
+interface BaseWarpRouteMetricLabels {
+  chain_name: ChainName;
+  token_address: string;
+  token_name: string;
+  warp_route_id: string;
+}
+
+interface WarpRouteMetricLabels extends BaseWarpRouteMetricLabels {
+  wallet_address: string;
+  token_standard: SupportedTokenStandards;
+  related_chain_names: string;
+}
+
+const warpRouteMetricLabels: (keyof WarpRouteMetricLabels)[] = [
+  'chain_name',
+  'token_address',
+  'token_name',
+  'wallet_address',
+  'token_standard',
+  'warp_route_id',
+  'related_chain_names',
+];
+
+const warpRouteTokenBalance = new Gauge({
+  name: 'hyperlane_warp_route_token_balance',
+  help: 'HypERC20 token balance of a Warp Route',
+  registers: [metricsRegister],
+  labelNames: warpRouteMetricLabels,
+});
+
+const warpRouteCollateralValue = new Gauge({
+  name: 'hyperlane_warp_route_collateral_value',
+  help: 'Total value of collateral held in a HypERC20Collateral or HypNative contract of a Warp Route',
+  registers: [metricsRegister],
+  labelNames: warpRouteMetricLabels,
+});
+
+interface WarpRouteValueAtRiskMetricLabels extends BaseWarpRouteMetricLabels {
+  collateral_chain_name: ChainName;
+  collateral_token_standard: SupportedTokenStandards;
+}
+
+const warpRouteValueAtRiskMetricLabels: (keyof WarpRouteValueAtRiskMetricLabels)[] =
+  [
+    'chain_name',
+    'collateral_chain_name',
+    'token_address',
+    'token_name',
+    'collateral_token_standard',
+    'warp_route_id',
+  ];
+
+const warpRouteValueAtRisk = new Gauge({
+  name: 'hyperlane_warp_route_value_at_risk',
+  help: 'Value at risk on chain for a given Warp Route',
+  registers: [metricsRegister],
+  labelNames: warpRouteValueAtRiskMetricLabels,
+});
+
+export const rebalancerExecutionTotal = new Counter({
+  name: 'hyperlane_rebalancer_executions_total',
+  help: 'Total number of rebalance execution attempts.',
+  registers: [metricsRegister],
+  labelNames: ['warp_route_id', 'succeeded'],
+});
+
+export const rebalancerExecutionAmount = new Counter({
+  name: 'hyperlane_rebalancer_execution_amount',
+  help: 'Total amount of tokens rebalanced.',
+  registers: [metricsRegister],
+  labelNames: ['warp_route_id', 'origin', 'destination', 'token'],
+});
+
+export const rebalancerPollingErrorsTotal = new Counter({
+  name: 'hyperlane_rebalancer_polling_errors_total',
+  help: 'Total number of errors during the monitor polling phase.',
+  registers: [metricsRegister],
+  labelNames: ['warp_route_id'],
+});
+
+const walletBalanceGauge = new Gauge({
+  // Mirror the rust/main/ethers-prometheus `wallet_balance` gauge metric.
+  name: 'hyperlane_wallet_balance',
+  help: 'Current balance of a wallet for a token',
+  registers: [metricsRegister],
+  labelNames: [
+    'chain',
+    'wallet_address',
+    'wallet_name',
+    'token_address',
+    'token_symbol',
+    'token_name',
+  ],
+});
+
+const xERC20LimitsGauge = new Gauge({
+  name: 'hyperlane_xerc20_limits',
+  help: 'Current minting and burning limits of xERC20 tokens',
+  registers: [metricsRegister],
+  labelNames: [
+    'chain_name',
+    'limit_type',
+    'token_name',
+    'bridge_address',
+    'token_address',
+    'bridge_label',
+  ],
+});
+
+export function updateTokenBalanceMetrics(
+  warpCore: WarpCore,
+  token: Token,
+  balanceInfo: WarpRouteBalance,
+  warpRouteId: string,
+  logger: Logger,
+) {
+  const allChains = warpCore.getTokenChains().sort();
+  const relatedChains = allChains.filter(
+    (chainName) => chainName !== token.chainName,
+  );
+
+  const metrics: WarpRouteMetricLabels = {
+    chain_name: token.chainName,
+    token_address: balanceInfo.tokenAddress,
+    token_name: token.name,
+    wallet_address:
+      // the balance for an EvmHypERC20 token is returned as the total supply of the xERC20 token,
+      // therefore we set the wallet address to the token address,
+      // we follow the same pattern or synthetic tokens
+      token.standard !== TokenStandard.EvmHypXERC20
+        ? token.addressOrDenom
+        : balanceInfo.tokenAddress,
+    token_standard:
+      // as we are reporting the total supply for clarity we report the standard as xERC20
+      token.standard !== TokenStandard.EvmHypXERC20 ? token.standard : 'xERC20',
+    warp_route_id: warpRouteId,
+    // TODO: consider deprecating this label given that we have the value at risk metric
+    related_chain_names: relatedChains.join(','),
+  };
+
+  warpRouteTokenBalance.labels(metrics).set(balanceInfo.balance);
+  logger.info(
+    {
+      labels: metrics,
+      balance: balanceInfo.balance,
+    },
+    'Wallet balance updated for token',
+  );
+
+  if (balanceInfo.valueUSD) {
+    // TODO: consider deprecating this metric in favor of the value at risk metric
+    warpRouteCollateralValue.labels(metrics).set(balanceInfo.valueUSD);
+    logger.info(
+      {
+        labels: metrics,
+        valueUSD: balanceInfo.valueUSD,
+      },
+      'Wallet value updated for token',
+    );
+
+    for (const chainName of allChains) {
+      const labels = {
+        chain_name: chainName,
+        collateral_chain_name: metrics.chain_name,
+        token_address: metrics.token_address,
+        token_name: metrics.token_name,
+        collateral_token_standard: metrics.token_standard,
+        warp_route_id: metrics.warp_route_id,
+      };
+
+      warpRouteValueAtRisk.labels(labels).set(balanceInfo.valueUSD);
+      logger.info(
+        {
+          labels,
+          valueUSD: balanceInfo.valueUSD,
+        },
+        `Value at risk on ${chainName} updated for token`,
+      );
+    }
+  }
+}
+// TODO: This does not need to be a separate function, we can redefine updateTokenBalanceMetrics to be generic
+// TODO: Consider adding some identifier for the managedLockbox contract, could be adding collateralName label for lockboxes, this would help different manages lockboxes that has a different collateral token
+export function updateManagedLockboxBalanceMetrics(
+  warpCore: WarpCore,
+  chainName: ChainName,
+  tokenName: string,
+  tokenAddress: string,
+  lockBoxAddress: string,
+  balanceInfo: WarpRouteBalance,
+  warpRouteId: string,
+  logger: Logger,
+) {
+  const metrics: WarpRouteMetricLabels = {
+    chain_name: chainName,
+    token_address: tokenAddress,
+    token_name: tokenName,
+    wallet_address: lockBoxAddress,
+    token_standard: 'EvmManagedLockbox', // TODO: we should eventually a new TokenStandard for this
+    warp_route_id: warpRouteId,
+    related_chain_names: warpCore
+      .getTokenChains()
+      .filter((_chainName) => _chainName !== chainName)
+      .sort()
+      .join(','),
+  };
+
+  warpRouteTokenBalance.labels(metrics).set(balanceInfo.balance);
+  logger.info(
+    {
+      labels: metrics,
+      balance: balanceInfo.balance,
+    },
+    'ManagedLockbox collateral balance updated',
+  );
+
+  if (balanceInfo.valueUSD) {
+    warpRouteCollateralValue.labels(metrics).set(balanceInfo.valueUSD);
+    logger.info(
+      {
+        labels: metrics,
+        valueUSD: balanceInfo.valueUSD,
+      },
+      'ManagedLockbox value updated for token',
+    );
+  }
+}
+
+export function updateNativeWalletBalanceMetrics(
+  balance: NativeWalletBalance,
+  logger: Logger,
+) {
+  walletBalanceGauge
+    .labels({
+      chain: balance.chain,
+      wallet_address: balance.walletAddress,
+      wallet_name: balance.walletName,
+      token_symbol: 'Native',
+      token_name: 'Native',
+    })
+    .set(balance.balance);
+  logger.info(
+    {
+      balanceInfo: balance,
+    },
+    'Native wallet balance updated',
+  );
+}
+
+export function updateXERC20LimitsMetrics(
+  token: Token,
+  limits: XERC20Limit,
+  bridgeAddress: Address,
+  bridgeLabel: string,
+  xERC20Address: Address,
+  logger: Logger,
+) {
+  const labels = {
+    chain_name: token.chainName,
+    token_name: token.name,
+    bridge_address: bridgeAddress,
+    token_address: xERC20Address,
+    bridge_label: bridgeLabel,
+  };
+
+  for (const [limitType, limit] of Object.entries(limits)) {
+    xERC20LimitsGauge
+      .labels({
+        ...labels,
+        limit_type: limitType,
+      })
+      .set(limit);
+  }
+
+  logger.info(
+    {
+      ...labels,
+      limits,
+    },
+    'xERC20 limits updated for bridge on token',
+  );
+}

package/src/metrics/types.ts

@@ -0,0 +1,27 @@
+import { ChainName } from '@hyperlane-xyz/sdk';
+import { Address } from '@hyperlane-xyz/utils';
+
+export interface XERC20Limit {
+  mint: number;
+  burn: number;
+  mintMax: number;
+  burnMax: number;
+}
+
+export interface XERC20Info {
+  limits: XERC20Limit;
+  xERC20Address: Address;
+}
+
+export interface WarpRouteBalance {
+  balance: number;
+  valueUSD?: number;
+  tokenAddress: Address;
+}
+
+export interface NativeWalletBalance {
+  chain: ChainName;
+  walletAddress: Address;
+  walletName: string;
+  balance: number;
+}

package/src/metrics/utils/metrics.ts

@@ -0,0 +1,33 @@
+import http from 'http';
+import { Registry } from 'prom-client';
+
+import { rootLogger } from '@hyperlane-xyz/utils';
+
+const logger = rootLogger.child({ module: 'metrics' });
+
+/**
+ * Start a simple HTTP server to host metrics. This just takes the registry and dumps the text
+ * string to people who request `GET /metrics`.
+ *
+ * PROMETHEUS_PORT env var is used to determine what port to host on, defaults to 9090.
+ */
+export function startMetricsServer(register: Registry): http.Server {
+  return http
+    .createServer((req, res) => {
+      if (req.url != '/metrics') {
+        return res.writeHead(404, 'Invalid url').end();
+      }
+
+      if (req.method != 'GET') {
+        return res.writeHead(405, 'Invalid method').end();
+      }
+
+      return register
+        .metrics()
+        .then((metricsStr) => {
+          res.writeHead(200, { 'Content-Type': 'text/plain' }).end(metricsStr);
+        })
+        .catch((err) => logger.error(err));
+    })
+    .listen(parseInt(process.env['PROMETHEUS_PORT'] || '9090'));
+}

package/src/monitor/Monitor.ts

@@ -0,0 +1,174 @@
+import EventEmitter from 'events';
+import { Logger } from 'pino';
+
+import type { Token, WarpCore } from '@hyperlane-xyz/sdk';
+import { sleep } from '@hyperlane-xyz/utils';
+
+import {
+  type IMonitor,
+  type MonitorEvent,
+  MonitorEventType,
+  MonitorPollingError,
+  MonitorStartError,
+} from '../interfaces/IMonitor.js';
+
+/**
+ * Simple monitor implementation that polls warp route collateral balances and emits them as MonitorEvent.
+ */
+export class Monitor implements IMonitor {
+  private readonly emitter = new EventEmitter();
+  private isMonitorRunning = false;
+  private resolveStop: (() => void) | null = null;
+  private stopPromise: Promise<void> | null = null;
+
+  /**
+   * @param checkFrequency - The frequency to poll balances in ms.
+   */
+  constructor(
+    private readonly checkFrequency: number,
+    private readonly warpCore: WarpCore,
+    private readonly logger: Logger,
+  ) {}
+
+  // overloads from IMonitor
+  on(
+    eventName: MonitorEventType.TokenInfo,
+    fn: (event: MonitorEvent) => void,
+  ): this;
+  on(eventName: MonitorEventType.Error, fn: (event: Error) => void): this;
+  on(eventName: MonitorEventType.Start, fn: () => void): this;
+  on(eventName: string, fn: (...args: any[]) => void): this {
+    this.emitter.on(eventName, fn);
+    return this;
+  }
+
+  async start() {
+    if (this.isMonitorRunning) {
+      // Cannot start the same monitor multiple times
+      this.emitter.emit(
+        MonitorEventType.Error,
+        new MonitorStartError('Monitor already running'),
+      );
+      return;
+    }
+
+    try {
+      this.isMonitorRunning = true;
+      this.logger.debug(
+        { checkFrequency: this.checkFrequency },
+        'Monitor started',
+      );
+      this.emitter.emit(MonitorEventType.Start);
+
+      while (this.isMonitorRunning) {
+        try {
+          this.logger.debug('Polling cycle started');
+          const event: MonitorEvent = {
+            tokensInfo: [],
+          };
+
+          for (const token of this.warpCore.tokens) {
+            this.logger.debug(
+              {
+                chain: token.chainName,
+                tokenSymbol: token.symbol,
+                tokenAddress: token.addressOrDenom,
+              },
+              'Checking token',
+            );
+            const bridgedSupply = await this.getTokenBridgedSupply(token);
+
+            event.tokensInfo.push({
+              token,
+              bridgedSupply,
+            });
+          }
+
+          // Emit the event warp routes info
+          this.emitter.emit(MonitorEventType.TokenInfo, event);
+          this.logger.debug('Polling cycle completed');
+        } catch (error) {
+          this.emitter.emit(
+            MonitorEventType.Error,
+            new MonitorPollingError(
+              `Error during monitor execution cycle: ${(error as Error).message}`,
+              error as Error,
+            ),
+          );
+        }
+
+        // Wait for the specified check frequency before the next iteration
+        await sleep(this.checkFrequency);
+      }
+    } catch (error) {
+      this.emitter.emit(
+        MonitorEventType.Error,
+        new MonitorStartError(
+          `Error starting monitor: ${(error as Error).message}`,
+          error as Error,
+        ),
+      );
+    }
+
+    // After the loop has been gracefully terminated, we can clean up.
+    this.emitter.removeAllListeners();
+    this.logger.info('Monitor stopped');
+
+    // If stop() was called, resolve the promise to signal that we're done.
+    if (this.resolveStop) {
+      this.resolveStop();
+      this.resolveStop = null;
+      this.stopPromise = null;
+    }
+  }
+
+  private async getTokenBridgedSupply(
+    token: Token,
+  ): Promise<bigint | undefined> {
+    if (!token.isHypToken()) {
+      this.logger.warn(
+        {
+          chain: token.chainName,
+          tokenSymbol: token.symbol,
+          tokenAddress: token.addressOrDenom,
+        },
+        'Cannot get bridged balance for a non-Hyperlane token',
+      );
+      return;
+    }
+
+    const adapter = token.getHypAdapter(this.warpCore.multiProvider);
+    const bridgedSupply = await adapter.getBridgedSupply();
+
+    if (bridgedSupply === undefined) {
+      this.logger.warn(
+        {
+          chain: token.chainName,
+          tokenSymbol: token.symbol,
+          tokenAddress: token.addressOrDenom,
+        },
+        'Bridged supply not found for token',
+      );
+    }
+
+    return bridgedSupply;
+  }
+
+  stop(): Promise<void> {
+    if (!this.isMonitorRunning) return Promise.resolve();
+
+    // If stop is already in progress, return the existing promise
+    if (this.stopPromise) return this.stopPromise;
+
+    this.logger.info('Stopping monitor...');
+    // Signal the while loop to terminate after its current iteration
+    this.isMonitorRunning = false;
+
+    // Create a promise that will be resolved by the start() method
+    // once the loop and cleanup are complete.
+    this.stopPromise = new Promise((resolve) => {
+      this.resolveStop = resolve;
+    });
+    return this.stopPromise;
+  }
+}

package/src/service.ts

@@ -0,0 +1,154 @@
+#!/usr/bin/env node
+/**
+ * Hyperlane Rebalancer Service Entry Point
+ *
+ * This is the main entry point for running the rebalancer as a standalone service
+ * in Kubernetes or other container environments. It reads configuration from
+ * environment variables and files, then starts the rebalancer in daemon mode.
+ *
+ * Environment Variables:
+ * - REBALANCER_CONFIG_FILE: Path to the rebalancer configuration YAML file (required)
+ * - HYP_KEY: Private key for signing transactions (required)
+ * - COINGECKO_API_KEY: API key for CoinGecko price fetching (optional, for metrics)
+ * - CHECK_FREQUENCY: Balance check frequency in ms (default: 60000)
+ * - WITH_METRICS: Enable Prometheus metrics (default: "true")
+ * - MONITOR_ONLY: Run in monitor-only mode without executing transactions (default: "false")
+ * - LOG_LEVEL: Logging level (default: "info") - supported by pino
+ *
+ * Usage:
+ *   node dist/service.js
+ *   REBALANCER_CONFIG_FILE=/config/rebalancer.yaml HYP_KEY=0x... node dist/service.js
+ */
+import { Wallet } from 'ethers';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import { getRegistry } from '@hyperlane-xyz/registry/fs';
+import { MultiProtocolProvider, MultiProvider } from '@hyperlane-xyz/sdk';
+import { createServiceLogger, rootLogger } from '@hyperlane-xyz/utils';
+
+import { RebalancerConfig } from './config/RebalancerConfig.js';
+import { RebalancerService } from './core/RebalancerService.js';
+
+function getVersion(): string {
+  try {
+    const __dirname = fileURLToPath(new URL('.', import.meta.url));
+    const packageJson = JSON.parse(
+      fs.readFileSync(path.join(__dirname, '../package.json'), 'utf-8'),
+    );
+    return packageJson.version;
+  } catch (error) {
+    rootLogger.warn({ error }, 'Could not read version from package.json');
+    return 'unknown';
+  }
+}
+
+async function main(): Promise<void> {
+  const VERSION = getVersion();
+  // Validate required environment variables
+  const configFile = process.env.REBALANCER_CONFIG_FILE;
+  if (!configFile) {
+    rootLogger.error('REBALANCER_CONFIG_FILE environment variable is required');
+    process.exit(1);
+  }
+
+  const privateKey = process.env.HYP_KEY;
+  if (!privateKey) {
+    rootLogger.error('HYP_KEY environment variable is required');
+    process.exit(1);
+  }
+
+  // Parse optional environment variables
+  let checkFrequency = 60_000;
+  if (process.env.CHECK_FREQUENCY) {
+    const parsed = parseInt(process.env.CHECK_FREQUENCY, 10);
+    if (isNaN(parsed) || parsed <= 0) {
+      rootLogger.error(
+        'CHECK_FREQUENCY must be a positive number (milliseconds)',
+      );
+      process.exit(1);
+    }
+    checkFrequency = parsed;
+  }
+
+  const withMetrics = process.env.WITH_METRICS !== 'false';
+  const monitorOnly = process.env.MONITOR_ONLY === 'true';
+  const coingeckoApiKey = process.env.COINGECKO_API_KEY;
+
+  // Create logger (uses LOG_LEVEL environment variable for level configuration)
+  const logger = await createServiceLogger({
+    service: 'rebalancer',
+    version: VERSION,
+  });
+
+  logger.info(
+    {
+      version: VERSION,
+      configFile,
+      checkFrequency,
+      withMetrics,
+      monitorOnly,
+    },
+    'Starting Hyperlane Rebalancer Service',
+  );
+
+  try {
+    // Load rebalancer configuration
+    const rebalancerConfig = RebalancerConfig.load(configFile);
+    logger.info('✅ Loaded rebalancer configuration');
+
+    // Initialize registry (uses default registry URIs)
+    const registry = getRegistry({
+      registryUris: [],
+      enableProxy: false,
+      logger: rootLogger,
+    });
+    logger.info('✅ Initialized registry');
+
+    // Get chain metadata from registry
+    const chainMetadata = await registry.getMetadata();
+    logger.info(
+      `✅ Loaded metadata for ${Object.keys(chainMetadata).length} chains`,
+    );
+
+    // Create MultiProvider with signer
+    const multiProvider = new MultiProvider(chainMetadata);
+    const signer = new Wallet(privateKey);
+    multiProvider.setSharedSigner(signer);
+    logger.info('✅ Initialized MultiProvider with signer');
+
+    // Create MultiProtocolProvider
+    const multiProtocolProvider = new MultiProtocolProvider(chainMetadata);
+    logger.info('✅ Initialized MultiProtocolProvider');
+
+    // Create the rebalancer service
+    const service = new RebalancerService(
+      multiProvider,
+      multiProtocolProvider,
+      registry,
+      rebalancerConfig,
+      {
+        mode: 'daemon',
+        checkFrequency,
+        monitorOnly,
+        withMetrics,
+        coingeckoApiKey,
+        logger,
+        version: VERSION,
+      },
+    );
+
+    // Start the service
+    await service.start();
+  } catch (error) {
+    logger.error({ error }, 'Failed to start rebalancer service');
+    process.exit(1);
+  }
+}
+
+// Run the service
+main().catch((error) => {
+  rootLogger.error({ error }, 'Fatal error');
+  process.exit(1);
+});