@hyperlane-xyz/rebalancer-sim 0.1.1

package/src/KPICollector.ts

@@ -0,0 +1,304 @@
+import type { ethers } from 'ethers';
+
+import { ERC20Test__factory } from '@hyperlane-xyz/core';
+
+import type {
+  ChainMetrics,
+  DeployedDomain,
+  RebalanceRecord,
+  SimulationKPIs,
+  TransferRecord,
+} from './types.js';
+
+/**
+ * KPICollector tracks metrics throughout a simulation run.
+ */
+export class KPICollector {
+  private transferRecords: Map<string, TransferRecord> = new Map();
+  private rebalanceRecords: Map<string, RebalanceRecord> = new Map();
+  /** Maps bridge transfer ID to rebalance ID for correlation */
+  private bridgeToRebalanceMap: Map<string, string> = new Map();
+  private initialBalances: Record<string, bigint> = {};
+
+  constructor(
+    private readonly provider: ethers.providers.JsonRpcProvider,
+    private readonly domains: Record<string, DeployedDomain>,
+  ) {}
+
+  /**
+   * Initialize with initial balances (passed explicitly or fetched)
+   */
+  async initialize(initialBalances?: Record<string, bigint>): Promise<void> {
+    if (initialBalances) {
+      this.initialBalances = { ...initialBalances };
+    } else {
+      for (const chainName of Object.keys(this.domains)) {
+        this.initialBalances[chainName] = await this.getBalance(chainName);
+      }
+    }
+  }
+
+  /**
+   * Get current balance for a chain's warp token
+   */
+  private async getBalance(chainName: string): Promise<bigint> {
+    const domain = this.domains[chainName];
+    const token = ERC20Test__factory.connect(
+      domain.collateralToken,
+      this.provider,
+    );
+    const balance = await token.balanceOf(domain.warpToken);
+    return balance.toBigInt();
+  }
+
+  /**
+   * Record transfer start
+   */
+  recordTransferStart(
+    id: string,
+    origin: string,
+    destination: string,
+    amount: bigint,
+  ): void {
+    this.transferRecords.set(id, {
+      id,
+      origin,
+      destination,
+      amount,
+      startTime: Date.now(),
+      status: 'pending',
+    });
+  }
+
+  /**
+   * Record transfer completion
+   */
+  recordTransferComplete(id: string): void {
+    const record = this.transferRecords.get(id);
+    if (record) {
+      record.endTime = Date.now();
+      record.latency = record.endTime - record.startTime;
+      record.status = 'completed';
+    }
+  }
+
+  /**
+   * Record transfer failure
+   */
+  recordTransferFailed(id: string): void {
+    const record = this.transferRecords.get(id);
+    if (record) {
+      record.endTime = Date.now();
+      record.status = 'failed';
+    }
+  }
+
+  /**
+   * Mark all pending transfers as complete (used after mailbox processing)
+   */
+  markAllPendingAsComplete(): void {
+    const now = Date.now();
+    for (const record of this.transferRecords.values()) {
+      if (record.status === 'pending') {
+        record.endTime = now;
+        record.latency = now - record.startTime;
+        record.status = 'completed';
+      }
+    }
+  }
+
+  /**
+   * Record a rebalance operation start (when SentTransferRemote fires)
+   * Returns the rebalance ID for correlation
+   */
+  recordRebalanceStart(
+    origin: string,
+    destination: string,
+    amount: bigint,
+    gasCost: bigint,
+  ): string {
+    const id = `rebalance-${this.rebalanceRecords.size}`;
+    this.rebalanceRecords.set(id, {
+      id,
+      origin,
+      destination,
+      amount,
+      startTime: Date.now(),
+      gasCost,
+      status: 'pending',
+    });
+    return id;
+  }
+
+  /**
+   * Link a bridge transfer ID to a rebalance ID for delivery tracking
+   */
+  linkBridgeTransfer(bridgeTransferId: string, rebalanceId: string): void {
+    this.bridgeToRebalanceMap.set(bridgeTransferId, rebalanceId);
+    const record = this.rebalanceRecords.get(rebalanceId);
+    if (record) {
+      record.bridgeTransferId = bridgeTransferId;
+    }
+  }
+
+  /**
+   * Record rebalance completion (when bridge delivers)
+   */
+  recordRebalanceComplete(bridgeTransferId: string): void {
+    const rebalanceId = this.bridgeToRebalanceMap.get(bridgeTransferId);
+    if (!rebalanceId) return;
+
+    const record = this.rebalanceRecords.get(rebalanceId);
+    if (record && record.status === 'pending') {
+      record.endTime = Date.now();
+      record.latency = record.endTime - record.startTime;
+      record.status = 'completed';
+    }
+  }
+
+  /**
+   * Record rebalance failure
+   */
+  recordRebalanceFailed(bridgeTransferId: string): void {
+    const rebalanceId = this.bridgeToRebalanceMap.get(bridgeTransferId);
+    if (!rebalanceId) return;
+
+    const record = this.rebalanceRecords.get(rebalanceId);
+    if (record) {
+      record.endTime = Date.now();
+      record.status = 'failed';
+    }
+  }
+
+  /**
+   * Get pending rebalances count
+   */
+  getPendingRebalancesCount(): number {
+    return Array.from(this.rebalanceRecords.values()).filter(
+      (r) => r.status === 'pending',
+    ).length;
+  }
+
+  /**
+   * Calculate percentile from sorted array
+   */
+  private percentile(sorted: number[], p: number): number {
+    if (sorted.length === 0) return 0;
+    const index = Math.ceil((p / 100) * sorted.length) - 1;
+    return sorted[Math.max(0, index)];
+  }
+
+  /**
+   * Generate final KPIs
+   */
+  async generateKPIs(): Promise<SimulationKPIs> {
+    const transfers = Array.from(this.transferRecords.values());
+    const completed = transfers.filter((t) => t.status === 'completed');
+    const failed = transfers.filter((t) => t.status === 'failed');
+
+    // Calculate latencies
+    const latencies = completed
+      .filter((t) => t.latency !== undefined)
+      .map((t) => t.latency!)
+      .sort((a, b) => a - b);
+
+    const avgLatency =
+      latencies.length > 0
+        ? latencies.reduce((a, b) => a + b, 0) / latencies.length
+        : 0;
+
+    // Calculate per-chain metrics
+    const perChainMetrics: Record<string, ChainMetrics> = {};
+    for (const chainName of Object.keys(this.domains)) {
+      const transfersIn = transfers.filter(
+        (t) => t.destination === chainName && t.status === 'completed',
+      ).length;
+      const transfersOut = transfers.filter(
+        (t) => t.origin === chainName && t.status === 'completed',
+      ).length;
+
+      const allRebalances = Array.from(this.rebalanceRecords.values());
+      const rebalancesIn = allRebalances.filter(
+        (r) => r.destination === chainName && r.status === 'completed',
+      ).length;
+      const rebalancesOut = allRebalances.filter(
+        (r) => r.origin === chainName && r.status === 'completed',
+      ).length;
+
+      const rebalanceVolumeIn = allRebalances
+        .filter((r) => r.destination === chainName && r.status === 'completed')
+        .reduce((sum, r) => sum + r.amount, BigInt(0));
+      const rebalanceVolumeOut = allRebalances
+        .filter((r) => r.origin === chainName && r.status === 'completed')
+        .reduce((sum, r) => sum + r.amount, BigInt(0));
+
+      const finalBalance = await this.getBalance(chainName);
+
+      perChainMetrics[chainName] = {
+        chainName,
+        initialBalance: this.initialBalances[chainName] ?? BigInt(0),
+        finalBalance,
+        transfersIn,
+        transfersOut,
+        rebalancesIn,
+        rebalancesOut,
+        rebalanceVolumeIn,
+        rebalanceVolumeOut,
+      };
+    }
+
+    // Calculate rebalance totals
+    const allRebalanceRecords = Array.from(this.rebalanceRecords.values());
+    const completedRebalances = allRebalanceRecords.filter(
+      (r) => r.status === 'completed',
+    );
+    const totalRebalanceVolume = completedRebalances.reduce(
+      (sum, r) => sum + r.amount,
+      BigInt(0),
+    );
+    const totalGasCost = completedRebalances.reduce(
+      (sum, r) => sum + r.gasCost,
+      BigInt(0),
+    );
+
+    return {
+      totalTransfers: transfers.length,
+      completedTransfers: completed.length,
+      failedTransfers: failed.length,
+      completionRate:
+        transfers.length > 0 ? completed.length / transfers.length : 1,
+      averageLatency: avgLatency,
+      p50Latency: this.percentile(latencies, 50),
+      p95Latency: this.percentile(latencies, 95),
+      p99Latency: this.percentile(latencies, 99),
+      totalRebalances: completedRebalances.length,
+      rebalanceVolume: totalRebalanceVolume,
+      totalGasCost,
+      perChainMetrics,
+    };
+  }
+
+  /**
+   * Get transfer records
+   */
+  getTransferRecords(): TransferRecord[] {
+    return Array.from(this.transferRecords.values());
+  }
+
+  /**
+   * Get rebalance records
+   */
+  getRebalanceRecords(): RebalanceRecord[] {
+    return Array.from(this.rebalanceRecords.values());
+  }
+
+  /**
+   * Reset collector for new simulation
+   */
+  reset(): void {
+    this.transferRecords.clear();
+    this.rebalanceRecords.clear();
+    this.bridgeToRebalanceMap.clear();
+    this.initialBalances = {};
+  }
+}

package/src/MessageTracker.ts

@@ -0,0 +1,312 @@
+import { ethers } from 'ethers';
+import { EventEmitter } from 'events';
+
+import { MockMailbox__factory } from '@hyperlane-xyz/core';
+import { rootLogger } from '@hyperlane-xyz/utils';
+
+import type { DeployedDomain } from './types.js';
+
+const logger = rootLogger.child({ module: 'MessageTracker' });
+
+/**
+ * Tracked message for off-chain processing control
+ */
+export interface TrackedMessage {
+  id: string;
+  transferId: string;
+  origin: string;
+  destination: string;
+  /** Nonce on the destination mailbox */
+  destinationNonce: number;
+  /** When the message was dispatched */
+  dispatchedAt: number;
+  /** When we should attempt delivery */
+  deliveryTime: number;
+  /** Processing status */
+  status: 'pending' | 'inflight' | 'delivered' | 'failed';
+  /** Number of delivery attempts */
+  attempts: number;
+  /** Last error if failed */
+  lastError?: string;
+}
+
+/**
+ * MessageTracker provides off-chain tracking and selective processing
+ * of Hyperlane messages. Fires transactions in parallel without blocking
+ * on receipts, similar to how the Hyperlane relayer batches messages.
+ */
+export class MessageTracker extends EventEmitter {
+  private messages: Map<string, TrackedMessage> = new Map();
+  private messageCounter = 0;
+  private destinationNonces: Map<string, number> = new Map();
+  private signer: ethers.Wallet;
+  private currentNonce: number = 0;
+  private nonceInitialized = false;
+
+  constructor(
+    private readonly provider: ethers.providers.JsonRpcProvider,
+    private readonly domains: Record<string, DeployedDomain>,
+    signerKey: string,
+  ) {
+    super();
+    this.signer = new ethers.Wallet(signerKey, provider);
+  }
+
+  /**
+   * Initialize by fetching current nonces from all destination mailboxes
+   */
+  async initialize(): Promise<void> {
+    for (const [chainName, domain] of Object.entries(this.domains)) {
+      const mailbox = MockMailbox__factory.connect(
+        domain.mailbox,
+        this.provider,
+      );
+      const nonce = await mailbox.inboundUnprocessedNonce();
+      this.destinationNonces.set(chainName, Number(nonce));
+    }
+    // Initialize signer nonce for parallel tx submission
+    this.currentNonce = await this.signer.getTransactionCount();
+    this.nonceInitialized = true;
+  }
+
+  /**
+   * Track a new message after a transfer is initiated.
+   * Call this after transferRemote() succeeds.
+   */
+  async trackMessage(
+    transferId: string,
+    origin: string,
+    destination: string,
+    deliveryDelay: number,
+  ): Promise<TrackedMessage> {
+    const destDomain = this.domains[destination];
+    const mailbox = MockMailbox__factory.connect(
+      destDomain.mailbox,
+      this.provider,
+    );
+    await mailbox.inboundUnprocessedNonce(); // Verify mailbox is accessible
+
+    const expectedNonce = this.destinationNonces.get(destination) || 0;
+    this.destinationNonces.set(destination, expectedNonce + 1);
+
+    const message: TrackedMessage = {
+      id: `msg-${this.messageCounter++}`,
+      transferId,
+      origin,
+      destination,
+      destinationNonce: expectedNonce,
+      dispatchedAt: Date.now(),
+      deliveryTime: Date.now() + deliveryDelay,
+      status: 'pending',
+      attempts: 0,
+    };
+
+    this.messages.set(message.id, message);
+    this.emit('message_tracked', message);
+
+    return message;
+  }
+
+  /**
+   * Get all messages ready for delivery (past their delivery time, not inflight)
+   */
+  getReadyMessages(): TrackedMessage[] {
+    const now = Date.now();
+    return Array.from(this.messages.values()).filter(
+      (m) => m.status === 'pending' && m.deliveryTime <= now,
+    );
+  }
+
+  /**
+   * Get all pending messages (including not yet ready and inflight)
+   */
+  getPendingMessages(): TrackedMessage[] {
+    return Array.from(this.messages.values()).filter(
+      (m) => m.status === 'pending' || m.status === 'inflight',
+    );
+  }
+
+  /**
+   * Process all ready messages in parallel without blocking on receipts.
+   * Fires transactions and subscribes to completion asynchronously.
+   */
+  async processReadyMessages(): Promise<{ delivered: number; failed: number }> {
+    const ready = this.getReadyMessages();
+    if (ready.length === 0) {
+      return { delivered: 0, failed: 0 };
+    }
+
+    // Ensure nonce is initialized
+    if (!this.nonceInitialized) {
+      this.currentNonce = await this.signer.getTransactionCount();
+      this.nonceInitialized = true;
+    }
+
+    // Check which messages can actually be processed (have sufficient liquidity)
+    // by doing a static call first
+    const processable: TrackedMessage[] = [];
+
+    const checkStartTime = Date.now();
+    for (const message of ready) {
+      const destDomain = this.domains[message.destination];
+      const mailbox = MockMailbox__factory.connect(
+        destDomain.mailbox,
+        this.signer,
+      );
+
+      const staticCallStart = Date.now();
+      try {
+        // Static call to check if it would succeed
+        await mailbox.callStatic.processInboundMessage(
+          message.destinationNonce,
+        );
+        const staticCallDuration = Date.now() - staticCallStart;
+        if (staticCallDuration > 100) {
+          logger.warn(
+            { transferId: message.transferId, staticCallDuration },
+            'Slow static call',
+          );
+        }
+        processable.push(message);
+        // Log successful processing after retries
+        if (message.attempts > 0) {
+          const waitTime = Date.now() - message.dispatchedAt;
+          logger.debug(
+            {
+              transferId: message.transferId,
+              origin: message.origin,
+              destination: message.destination,
+              attempts: message.attempts,
+              waitTime,
+            },
+            'Message ready after retries',
+          );
+        }
+      } catch (error: any) {
+        const staticCallDuration = Date.now() - staticCallStart;
+        const errorMsg = error.reason || error.message || '';
+        // Check if message was already delivered (e.g., by bridge controller)
+        // This is a permanent state, not a temporary error
+        if (errorMsg.includes('already delivered')) {
+          message.status = 'delivered';
+          this.emit('message_delivered', message);
+          continue;
+        }
+        // Other errors - mark attempt but keep pending for retry
+        message.attempts++;
+        message.lastError = errorMsg;
+
+        // Log failures - every 5 attempts or on slow static calls
+        if (message.attempts % 5 === 0 || staticCallDuration > 100) {
+          const waitTime = Date.now() - message.dispatchedAt;
+          logger.debug(
+            {
+              transferId: message.transferId,
+              origin: message.origin,
+              destination: message.destination,
+              attempts: message.attempts,
+              waitTime,
+              error: errorMsg,
+            },
+            'Message delivery failed, will retry',
+          );
+        }
+      }
+    }
+
+    const totalCheckTime = Date.now() - checkStartTime;
+    if (totalCheckTime > 500) {
+      logger.warn(
+        { messageCount: ready.length, totalCheckTime },
+        'Slow static call checks',
+      );
+    }
+
+    if (processable.length === 0) {
+      // No messages processable yet - not a failure, they will retry
+      return { delivered: 0, failed: 0 };
+    }
+
+    // Fire all processable transactions in parallel
+    const txPromises: Array<{
+      message: TrackedMessage;
+      txPromise: Promise<ethers.ContractTransaction>;
+    }> = [];
+
+    for (const message of processable) {
+      message.status = 'inflight';
+      message.attempts++;
+
+      const destDomain = this.domains[message.destination];
+      const mailbox = MockMailbox__factory.connect(
+        destDomain.mailbox,
+        this.signer,
+      );
+
+      // Fire transaction with explicit nonce (don't wait)
+      const txPromise = mailbox.processInboundMessage(
+        message.destinationNonce,
+        { nonce: this.currentNonce++ },
+      );
+
+      txPromises.push({ message, txPromise });
+    }
+
+    // Subscribe to all tx completions asynchronously
+    let delivered = 0;
+    let failed = 0;
+
+    await Promise.all(
+      txPromises.map(async ({ message, txPromise }) => {
+        try {
+          const tx = await txPromise;
+          await tx.wait();
+
+          message.status = 'delivered';
+          this.emit('message_delivered', message);
+          delivered++;
+        } catch (error: any) {
+          // Transaction failed - back to pending for retry
+          message.status = 'pending';
+          message.lastError = error.reason || error.message;
+          failed++;
+        }
+      }),
+    );
+
+    return { delivered, failed };
+  }
+
+  /**
+   * Check if there are any pending or inflight messages
+   */
+  hasPendingMessages(): boolean {
+    return this.getPendingMessages().length > 0;
+  }
+
+  /**
+   * Get message by transfer ID
+   */
+  getMessageByTransferId(transferId: string): TrackedMessage | undefined {
+    return Array.from(this.messages.values()).find(
+      (m) => m.transferId === transferId,
+    );
+  }
+
+  /**
+   * Get all messages
+   */
+  getAllMessages(): TrackedMessage[] {
+    return Array.from(this.messages.values());
+  }
+
+  /**
+   * Clear all tracked messages (for reset)
+   */
+  clear(): void {
+    this.messages.clear();
+    this.messageCounter = 0;
+    this.destinationNonces.clear();
+    this.nonceInitialized = false;
+  }
+}