@hyperlane-xyz/rebalancer 3.0.0 → 3.1.0

package/src/tracking/ActionTracker.test.ts

@@ -5,6 +5,7 @@ import Sinon from 'sinon';
 
 import { EthJsonRpcBlockParameterTag } from '@hyperlane-xyz/sdk';
 
+import { DEFAULT_INTENT_TTL_MS } from '../config/types.js';
 import type { ExplorerMessage } from '../utils/ExplorerClient.js';
 
 import { ActionTracker, type ActionTrackerConfig } from './ActionTracker.js';
@@ -71,6 +72,7 @@ describe('ActionTracker', () => {
       },
       bridges: ['0xbridge1', '0xbridge2'],
       rebalancerAddress: '0xrebalancer',
+      intentTTL: DEFAULT_INTENT_TTL_MS,
     };
 
     tracker = new ActionTracker(
@@ -99,6 +101,7 @@ describe('ActionTracker', () => {
           is_delivered: false,
           message_body:
             '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064',
+          send_occurred_at: null,
         },
       ];
 
@@ -128,6 +131,102 @@ describe('ActionTracker', () => {
       expect(actions[0].messageId).to.equal('0xmsg1');
     });
 
+    it('should use send_occurred_at for createdAt when available', async () => {
+      const inflightMessages: ExplorerMessage[] = [
+        {
+          msg_id: '0xmsg1',
+          origin_domain_id: 1,
+          destination_domain_id: 2,
+          sender: '0xrouter1',
+          recipient: '0xrouter2',
+          origin_tx_hash: '0xtx1',
+          origin_tx_sender: '0xrebalancer',
+          origin_tx_recipient: '0xrouter1',
+          is_delivered: false,
+          message_body:
+            '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064',
+          send_occurred_at: '2024-01-15T12:30:45',
+        },
+      ];
+
+      explorerClient.getInflightRebalanceActions.resolves(inflightMessages);
+      explorerClient.getInflightUserTransfers.resolves([]);
+      mailboxStub.isDelivered.resolves(false);
+
+      await tracker.initialize();
+
+      const intents = await rebalanceIntentStore.getAll();
+      expect(intents).to.have.lengthOf(1);
+      // Hasura timestamps are UTC without 'Z'; recoverAction appends 'Z' before parsing
+      const expectedMs = new Date('2024-01-15T12:30:45Z').getTime();
+      expect(intents[0].createdAt).to.equal(expectedMs);
+
+      const actions = await rebalanceActionStore.getAll();
+      expect(actions[0].createdAt).to.equal(expectedMs);
+    });
+
+    it('should fall back to Date.now() when send_occurred_at is null', async () => {
+      const before = Date.now();
+      const inflightMessages: ExplorerMessage[] = [
+        {
+          msg_id: '0xmsg1',
+          origin_domain_id: 1,
+          destination_domain_id: 2,
+          sender: '0xrouter1',
+          recipient: '0xrouter2',
+          origin_tx_hash: '0xtx1',
+          origin_tx_sender: '0xrebalancer',
+          origin_tx_recipient: '0xrouter1',
+          is_delivered: false,
+          message_body:
+            '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064',
+          send_occurred_at: null,
+        },
+      ];
+
+      explorerClient.getInflightRebalanceActions.resolves(inflightMessages);
+      explorerClient.getInflightUserTransfers.resolves([]);
+      mailboxStub.isDelivered.resolves(false);
+
+      await tracker.initialize();
+      const after = Date.now();
+
+      const intents = await rebalanceIntentStore.getAll();
+      expect(intents[0].createdAt).to.be.at.least(before);
+      expect(intents[0].createdAt).to.be.at.most(after);
+    });
+
+    it('should skip action with invalid send_occurred_at timestamp', async () => {
+      const inflightMessages: ExplorerMessage[] = [
+        {
+          msg_id: '0xmsg1',
+          origin_domain_id: 1,
+          destination_domain_id: 2,
+          sender: '0xrouter1',
+          recipient: '0xrouter2',
+          origin_tx_hash: '0xtx1',
+          origin_tx_sender: '0xrebalancer',
+          origin_tx_recipient: '0xrouter1',
+          is_delivered: false,
+          message_body:
+            '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064',
+          send_occurred_at: 'garbage',
+        },
+      ];
+
+      explorerClient.getInflightRebalanceActions.resolves(inflightMessages);
+      explorerClient.getInflightUserTransfers.resolves([]);
+      mailboxStub.isDelivered.resolves(false);
+
+      await tracker.initialize();
+
+      // Invalid timestamp is caught by recoverAction's catch block, so the action is skipped
+      const intents = await rebalanceIntentStore.getAll();
+      expect(intents).to.have.lengthOf(0);
+      const actions = await rebalanceActionStore.getAll();
+      expect(actions).to.have.lengthOf(0);
+    });
+
     it('should skip creating action if it already exists', async () => {
       const inflightMessages: ExplorerMessage[] = [
         {
@@ -142,6 +241,7 @@ describe('ActionTracker', () => {
           is_delivered: false,
           message_body:
             '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064',
+          send_occurred_at: null,
         },
       ];
 
@@ -189,6 +289,7 @@ describe('ActionTracker', () => {
           is_delivered: false,
           message_body:
             '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064',
+          send_occurred_at: null,
         },
       ];
 
@@ -232,6 +333,7 @@ describe('ActionTracker', () => {
           is_delivered: false,
           message_body:
             '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064',
+          send_occurred_at: null,
         },
       ];
 
@@ -336,6 +438,61 @@ describe('ActionTracker', () => {
       const updated = await rebalanceIntentStore.get('intent-1');
       expect(updated?.status).to.equal('in_progress');
     });
+
+    it('should mark unfulfilled intents as failed when TTL exceeded', async () => {
+      const intent: RebalanceIntent = {
+        id: 'intent-1',
+        status: 'in_progress',
+        origin: 1,
+        destination: 2,
+        amount: 100n,
+        createdAt: Date.now() - DEFAULT_INTENT_TTL_MS - 1,
+        updatedAt: Date.now(),
+      };
+
+      const action: RebalanceAction = {
+        id: 'action-1',
+        type: 'rebalance_message',
+        status: 'in_progress',
+        intentId: 'intent-1',
+        messageId: '0xmsg1',
+        origin: 1,
+        destination: 2,
+        amount: 50n,
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      };
+
+      await rebalanceIntentStore.save(intent);
+      await rebalanceActionStore.save(action);
+
+      await tracker.syncRebalanceIntents();
+
+      const updatedIntent = await rebalanceIntentStore.get('intent-1');
+      expect(updatedIntent?.status).to.equal('failed');
+
+      const updatedAction = await rebalanceActionStore.get('action-1');
+      expect(updatedAction?.status).to.equal('failed');
+    });
+
+    it('should not expire intents within TTL', async () => {
+      const intent: RebalanceIntent = {
+        id: 'intent-1',
+        status: 'in_progress',
+        origin: 1,
+        destination: 2,
+        amount: 100n,
+        createdAt: Date.now() - DEFAULT_INTENT_TTL_MS + 60_000,
+        updatedAt: Date.now(),
+      };
+
+      await rebalanceIntentStore.save(intent);
+
+      await tracker.syncRebalanceIntents();
+
+      const updated = await rebalanceIntentStore.get('intent-1');
+      expect(updated?.status).to.equal('in_progress');
+    });
   });
 
   describe('syncRebalanceActions', () => {
@@ -555,6 +712,170 @@ describe('ActionTracker', () => {
 
       expect(partialIntents).to.have.lengthOf(0);
     });
+
+    it('returns intent with in-flight deposit and sets hasInflightDeposit flag', async () => {
+      // Setup: in_progress inventory intent (amount: 1 ETH = 1_000_000_000_000_000_000n)
+      await rebalanceIntentStore.save({
+        id: 'intent-with-inflight',
+        status: 'in_progress',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n, // 1 ETH
+        executionMethod: 'inventory',
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      // Complete inventory_deposit action (amount: 400_000_000_000_000_000n = 0.4 ETH)
+      await rebalanceActionStore.save({
+        id: 'action-complete',
+        type: 'inventory_deposit',
+        status: 'complete',
+        intentId: 'intent-with-inflight',
+        origin: 1,
+        destination: 2,
+        amount: 400000000000000000n, // 0.4 ETH completed
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      // In_progress inventory_deposit action (amount: 300_000_000_000_000_000n = 0.3 ETH)
+      await rebalanceActionStore.save({
+        id: 'action-inflight',
+        type: 'inventory_deposit',
+        status: 'in_progress',
+        intentId: 'intent-with-inflight',
+        origin: 1,
+        destination: 2,
+        amount: 300000000000000000n, // 0.3 ETH in-flight
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      const partialIntents =
+        await tracker.getPartiallyFulfilledInventoryIntents();
+
+      expect(partialIntents).to.have.lengthOf(1);
+      expect(partialIntents[0].hasInflightDeposit).to.be.true;
+      expect(partialIntents[0].completedAmount).to.equal(400000000000000000n);
+      expect(partialIntents[0].remaining).to.equal(300000000000000000n); // 1.0 - 0.4 - 0.3
+    });
+
+    it('returns intent without in-flight deposit with hasInflightDeposit false', async () => {
+      // Setup: in_progress inventory intent (amount: 1 ETH)
+      await rebalanceIntentStore.save({
+        id: 'intent-no-inflight',
+        status: 'in_progress',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n, // 1 ETH
+        executionMethod: 'inventory',
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      // Complete inventory_deposit action (amount: 0.4 ETH)
+      await rebalanceActionStore.save({
+        id: 'action-complete-only',
+        type: 'inventory_deposit',
+        status: 'complete',
+        intentId: 'intent-no-inflight',
+        origin: 1,
+        destination: 2,
+        amount: 400000000000000000n, // 0.4 ETH completed
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+      // NO in_progress inventory_deposit actions
+
+      const partialIntents =
+        await tracker.getPartiallyFulfilledInventoryIntents();
+
+      expect(partialIntents).to.have.lengthOf(1);
+      expect(partialIntents[0].hasInflightDeposit).to.be.false;
+      expect(partialIntents[0].remaining).to.equal(600000000000000000n);
+    });
+
+    it('returns intent when remaining is 0n but has in-flight deposit', async () => {
+      // intent.amount = 1 ETH, completedAmount = 0.7 ETH, inflightAmount = 0.3 ETH → remaining = 0n
+      await rebalanceIntentStore.save({
+        id: 'intent-zero-remaining',
+        status: 'in_progress',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n, // 1 ETH
+        executionMethod: 'inventory',
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      // 0.7 ETH already completed
+      await rebalanceActionStore.save({
+        id: 'action-complete-part',
+        type: 'inventory_deposit',
+        status: 'complete',
+        intentId: 'intent-zero-remaining',
+        origin: 1,
+        destination: 2,
+        amount: 700000000000000000n, // 0.7 ETH
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      // 0.3 ETH in-flight — exactly fills the remaining gap
+      await rebalanceActionStore.save({
+        id: 'action-inflight-rest',
+        type: 'inventory_deposit',
+        status: 'in_progress',
+        intentId: 'intent-zero-remaining',
+        origin: 1,
+        destination: 2,
+        amount: 300000000000000000n, // 0.3 ETH in-flight
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      const partialIntents =
+        await tracker.getPartiallyFulfilledInventoryIntents();
+
+      expect(partialIntents).to.have.lengthOf(1);
+      expect(partialIntents[0].remaining).to.equal(0n);
+      expect(partialIntents[0].hasInflightDeposit).to.be.true;
+      expect(partialIntents[0].completedAmount).to.equal(700000000000000000n);
+    });
+
+    it('does not return fully completed intent with no in-flight deposits', async () => {
+      // intent.amount = 1 ETH, completedAmount = 1 ETH, inflightAmount = 0 → remaining = 0n, no inflight
+      await rebalanceIntentStore.save({
+        id: 'intent-fully-done',
+        status: 'in_progress',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n, // 1 ETH
+        executionMethod: 'inventory',
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      // Full amount completed
+      await rebalanceActionStore.save({
+        id: 'action-all-done',
+        type: 'inventory_deposit',
+        status: 'complete',
+        intentId: 'intent-fully-done',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n, // 1 ETH — full amount
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      const partialIntents =
+        await tracker.getPartiallyFulfilledInventoryIntents();
+
+      // remaining = 0n AND inflightAmount = 0n → should NOT be returned
+      expect(partialIntents).to.have.lengthOf(0);
+    });
   });
 
   describe('createRebalanceIntent', () => {

package/src/tracking/ActionTracker.ts

@@ -3,7 +3,7 @@ import { v4 as uuidv4 } from 'uuid';
 
 import type { MultiProtocolCore } from '@hyperlane-xyz/sdk';
 import type { Address, Domain } from '@hyperlane-xyz/utils';
-import { parseWarpRouteMessage } from '@hyperlane-xyz/utils';
+import { assert, parseWarpRouteMessage } from '@hyperlane-xyz/utils';
 
 import type { ExternalBridgeRegistry } from '../interfaces/IExternalBridge.js';
 import type { ConfirmedBlockTags } from '../interfaces/IMonitor.js';
@@ -34,6 +34,7 @@ export interface ActionTrackerConfig {
   bridges: Address[]; // Bridge contract addresses for rebalance action queries
   rebalancerAddress: Address;
   inventorySignerAddress?: Address; // Optional - for excluding inventory signer from user transfers query
+  intentTTL: number; // Max age in ms before in-progress intent is expired
 }
 
 /**
@@ -211,9 +212,12 @@ export class ActionTracker implements IActionTracker {
   async syncRebalanceIntents(): Promise<void> {
     this.logger.debug('Syncing rebalance intents');
 
-    // Check in_progress intents for completion by deriving from action states
+    // Check in_progress intents for completion or TTL expiry
     const inProgressIntents =
       await this.rebalanceIntentStore.getByStatus('in_progress');
+    const allInProgressActions =
+      await this.rebalanceActionStore.getByStatus('in_progress');
+    const now = Date.now();
     for (const intent of inProgressIntents) {
       const completedAmount = await this.getCompletedAmountForIntent(intent.id);
       if (completedAmount >= intent.amount) {
@@ -221,6 +225,39 @@ export class ActionTracker implements IActionTracker {
           status: 'complete',
         });
         this.logger.debug({ id: intent.id }, 'RebalanceIntent completed');
+      } else if (now - intent.createdAt > this.config.intentTTL) {
+        await this.rebalanceIntentStore.update(intent.id, {
+          status: 'failed',
+        });
+
+        // Fail any in-progress actions associated with the expired intent
+        for (const action of allInProgressActions) {
+          if (action.intentId === intent.id) {
+            await this.rebalanceActionStore.update(action.id, {
+              status: 'failed',
+            });
+            this.logger.warn(
+              { actionId: action.id, intentId: intent.id },
+              'RebalanceAction failed due to parent intent TTL expiry',
+            );
+          }
+        }
+
+        this.logger.debug(
+          {
+            id: intent.id,
+            origin: intent.origin,
+            destination: intent.destination,
+            amount: intent.amount.toString(),
+            ageMs: now - intent.createdAt,
+            ttlMs: this.config.intentTTL,
+          },
+          'RebalanceIntent TTL expiry details',
+        );
+        this.logger.warn(
+          { id: intent.id },
+          'RebalanceIntent expired due to TTL',
+        );
       }
     }
 
@@ -524,8 +561,8 @@ export class ActionTracker implements IActionTracker {
   }
 
   /**
-   * Get inventory intents that are in_progress or not_started but not fully fulfilled,
-   * and have no in-flight actions (safe to continue).
+   * Get inventory intents that are in_progress or not_started but not fully settled.
+   * Intents with in-flight deposits are included but flagged via hasInflightDeposit.
    * Returns enriched data with computed values derived from action states.
    *
    * NOTE: We include 'not_started' intents because they may have been created
@@ -580,9 +617,13 @@ export class ActionTracker implements IActionTracker {
 
       const remaining = intent.amount - completedAmount - inflightAmount;
 
-      // Safe to continue if: remaining > 0 AND no in-flight inventory_deposit
-      if (remaining > 0n && inflightAmount === 0n) {
-        partialIntents.push({ intent, completedAmount, remaining });
+      if (remaining > 0n || inflightAmount > 0n) {
+        partialIntents.push({
+          intent,
+          completedAmount,
+          remaining,
+          hasInflightDeposit: inflightAmount > 0n,
+        });
       }
     }
 
@@ -863,6 +904,16 @@ export class ActionTracker implements IActionTracker {
     try {
       // Create synthetic intent
       const { amount } = parseWarpRouteMessage(msg.message_body);
+      // Hasura returns block timestamps as UTC without 'Z' suffix (e.g. "2024-01-15T12:30:45").
+      // Null when the scraper hasn't indexed the origin block yet, so fall back to now.
+      // Note: when null, TTL effectively extends by scraper lag since recoverAction won't update createdAt later.
+      const createdAt = msg.send_occurred_at
+        ? new Date(msg.send_occurred_at + 'Z').getTime()
+        : Date.now();
+      assert(
+        !isNaN(createdAt),
+        `Invalid send_occurred_at timestamp: ${msg.send_occurred_at}`,
+      );
       const intent: RebalanceIntent = {
         id: uuidv4(),
         status: 'in_progress',
@@ -871,13 +922,13 @@ export class ActionTracker implements IActionTracker {
         amount,
         priority: undefined,
         strategyType: undefined,
-        createdAt: Date.now(),
+        createdAt,
         updatedAt: Date.now(),
       };
 
       await this.rebalanceIntentStore.save(intent);
       this.logger.debug(
-        { id: intent.id, amount: amount.toString() },
+        { id: intent.id, amount: amount.toString(), createdAt },
         'Created synthetic RebalanceIntent',
       );
 
@@ -892,7 +943,7 @@ export class ActionTracker implements IActionTracker {
         origin: msg.origin_domain_id,
         destination: msg.destination_domain_id,
         amount,
-        createdAt: Date.now(),
+        createdAt,
         updatedAt: Date.now(),
       };
 

package/src/tracking/IActionTracker.ts

@@ -119,8 +119,9 @@ export interface IActionTracker {
   ): Promise<RebalanceIntent[]>;
 
   /**
-   * Get inventory intents that are in_progress but not fully fulfilled,
-   * and have no in-flight actions (safe to continue).
+   * Returns inventory intents that have remaining work.
+   * Intents with in-flight deposits are included but marked via hasInflightDeposit —
+   * callers must check before continuing.
    * Returns enriched data with computed values derived from action states.
    */
   getPartiallyFulfilledInventoryIntents(): Promise<PartialInventoryIntent[]>;

package/src/tracking/types.ts

@@ -108,6 +108,8 @@ export interface PartialInventoryIntent {
   intent: RebalanceIntent;
   /** Sum of complete inventory_deposit action amounts */
   completedAmount: bigint;
-  /** Amount remaining to fulfill: intent.amount - completedAmount - inflightAmount */
+  /** Amount remaining to fulfill (0n when final deposit is fully in-flight). Formula: intent.amount - completedAmount - inflightAmount */
   remaining: bigint;
+  /** True when intent has in_progress inventory_deposit actions (not safe to continue, but still active) */
+  hasInflightDeposit: boolean;
 }

package/src/utils/ExplorerClient.ts

@@ -32,6 +32,7 @@ export type ExplorerMessage = {
   origin_tx_recipient: string;
   is_delivered: boolean;
   message_body: string;
+  send_occurred_at: string | null;
 };
 
 export interface IExplorerClient {
@@ -72,6 +73,7 @@ export class ExplorerClient implements IExplorerClient {
       origin_tx_recipient: normalizeHex(msg.origin_tx_recipient),
       is_delivered: msg.is_delivered,
       message_body: normalizeHex(msg.message_body),
+      send_occurred_at: msg.send_occurred_at ?? null,
     };
   }
 
@@ -242,6 +244,7 @@ export class ExplorerClient implements IExplorerClient {
           origin_tx_recipient
           is_delivered
           message_body
+          send_occurred_at
         }
       }`;
 
@@ -350,6 +353,7 @@ export class ExplorerClient implements IExplorerClient {
           origin_tx_recipient
           is_delivered
           message_body
+          send_occurred_at
         }
       }`;