@hyperlane-xyz/rebalancer 3.2.0 → 25.5.0

package/src/tracking/ActionTracker.test.ts

@@ -5,7 +5,11 @@ import Sinon from 'sinon';
 
 import { EthJsonRpcBlockParameterTag } from '@hyperlane-xyz/sdk';
 
-import { DEFAULT_INTENT_TTL_MS } from '../config/types.js';
+import {
+  DEFAULT_INTENT_TTL_MS,
+  DEFAULT_MOVEMENT_STALENESS_MS,
+  ExternalBridgeType,
+} from '../config/types.js';
 import type { ExplorerMessage } from '../utils/ExplorerClient.js';
 
 import { ActionTracker, type ActionTrackerConfig } from './ActionTracker.js';
@@ -561,6 +565,115 @@ describe('ActionTracker', () => {
     });
   });
 
+  describe('syncInventoryMovementActions', () => {
+    it('stores pending status on in-progress movement', async () => {
+      await rebalanceActionStore.save({
+        id: 'action-pending',
+        type: 'inventory_movement',
+        status: 'in_progress',
+        intentId: 'intent-1',
+        origin: 1,
+        destination: 2,
+        amount: 100n,
+        txHash: '0xtx-pending',
+        externalBridgeId: ExternalBridgeType.LiFi,
+        createdAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+        updatedAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+      });
+
+      const getStatus = Sinon.stub().resolves({ status: 'pending' });
+      await tracker.syncInventoryMovementActions({
+        lifi: { getStatus } as any,
+      });
+
+      const action = await rebalanceActionStore.get('action-pending');
+      expect(action?.status).to.equal('in_progress');
+      expect(action?.lastBridgeStatus).to.equal('pending');
+    });
+
+    it('stores not_found status on in-progress movement', async () => {
+      await rebalanceActionStore.save({
+        id: 'action-not-found',
+        type: 'inventory_movement',
+        status: 'in_progress',
+        intentId: 'intent-1',
+        origin: 1,
+        destination: 2,
+        amount: 100n,
+        txHash: '0xtx-not-found',
+        externalBridgeId: ExternalBridgeType.LiFi,
+        createdAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+        updatedAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+      });
+
+      const getStatus = Sinon.stub().resolves({ status: 'not_found' });
+      await tracker.syncInventoryMovementActions({
+        lifi: { getStatus } as any,
+      });
+
+      const action = await rebalanceActionStore.get('action-not-found');
+      expect(action?.status).to.equal('in_progress');
+      expect(action?.lastBridgeStatus).to.equal('not_found');
+      expect(action?.nonPendingSince).to.be.a('number');
+    });
+
+    it('clears nonPendingSince when status returns to pending', async () => {
+      await rebalanceActionStore.save({
+        id: 'action-back-to-pending',
+        type: 'inventory_movement',
+        status: 'in_progress',
+        intentId: 'intent-1',
+        origin: 1,
+        destination: 2,
+        amount: 100n,
+        txHash: '0xtx-back-pending',
+        externalBridgeId: ExternalBridgeType.LiFi,
+        lastBridgeStatus: 'not_found',
+        nonPendingSince: Date.now() - 60_000,
+        createdAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+        updatedAt: Date.now(),
+      });
+
+      const getStatus = Sinon.stub().resolves({ status: 'pending' });
+      await tracker.syncInventoryMovementActions({
+        lifi: { getStatus } as any,
+      });
+
+      const action = await rebalanceActionStore.get('action-back-to-pending');
+      expect(action?.lastBridgeStatus).to.equal('pending');
+      expect(action?.nonPendingSince).to.be.undefined;
+    });
+
+    it('preserves existing nonPendingSince on repeated not_found polls', async () => {
+      const originalNonPendingSince = Date.now() - 120_000;
+      await rebalanceActionStore.save({
+        id: 'action-repeated-not-found',
+        type: 'inventory_movement',
+        status: 'in_progress',
+        intentId: 'intent-1',
+        origin: 1,
+        destination: 2,
+        amount: 100n,
+        txHash: '0xtx-repeated',
+        externalBridgeId: ExternalBridgeType.LiFi,
+        lastBridgeStatus: 'not_found',
+        nonPendingSince: originalNonPendingSince,
+        createdAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+        updatedAt: Date.now(),
+      });
+
+      const getStatus = Sinon.stub().resolves({ status: 'not_found' });
+      await tracker.syncInventoryMovementActions({
+        lifi: { getStatus } as any,
+      });
+
+      const action = await rebalanceActionStore.get(
+        'action-repeated-not-found',
+      );
+      expect(action?.nonPendingSince).to.equal(originalNonPendingSince);
+    });
+  });
+
   describe('getInProgressTransfers', () => {
     it('should return only in_progress transfers', async () => {
       await transferStore.save({
@@ -876,6 +989,228 @@ describe('ActionTracker', () => {
       // remaining = 0n AND inflightAmount = 0n → should NOT be returned
       expect(partialIntents).to.have.lengthOf(0);
     });
+
+    it('skips intent with recent in-flight inventory_movement', async () => {
+      await rebalanceIntentStore.save({
+        id: 'intent-recent-movement',
+        status: 'in_progress',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n,
+        executionMethod: 'inventory',
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      // Recent movement (created just now)
+      await rebalanceActionStore.save({
+        id: 'movement-recent',
+        type: 'inventory_movement',
+        status: 'in_progress',
+        intentId: 'intent-recent-movement',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n,
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      const partialIntents =
+        await tracker.getPartiallyFulfilledInventoryIntents();
+      expect(partialIntents).to.have.lengthOf(0);
+    });
+
+    it('fails stale movement and returns intent', async () => {
+      await rebalanceIntentStore.save({
+        id: 'intent-stale-movement',
+        status: 'in_progress',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n,
+        executionMethod: 'inventory',
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      // Stale movement (non-pending for > 30 min)
+      await rebalanceActionStore.save({
+        id: 'movement-stale',
+        type: 'inventory_movement',
+        status: 'in_progress',
+        lastBridgeStatus: 'not_found',
+        nonPendingSince: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+        intentId: 'intent-stale-movement',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n,
+        createdAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+        updatedAt: Date.now(),
+      });
+
+      const partialIntents =
+        await tracker.getPartiallyFulfilledInventoryIntents();
+      expect(partialIntents).to.have.lengthOf(1);
+      expect(partialIntents[0].intent.id).to.equal('intent-stale-movement');
+
+      // Verify the stale movement was failed
+      const failedAction = await rebalanceActionStore.get('movement-stale');
+      expect(failedAction?.status).to.equal('failed');
+    });
+
+    it('fails stale movement with undefined lastBridgeStatus (pre-deploy data)', async () => {
+      await rebalanceIntentStore.save({
+        id: 'intent-undefined-status',
+        status: 'in_progress',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n,
+        executionMethod: 'inventory',
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      // Stale movement without lastBridgeStatus (simulates pre-deploy data)
+      await rebalanceActionStore.save({
+        id: 'movement-undefined-status',
+        type: 'inventory_movement',
+        status: 'in_progress',
+        intentId: 'intent-undefined-status',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n,
+        createdAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+        updatedAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+      });
+
+      const partialIntents =
+        await tracker.getPartiallyFulfilledInventoryIntents();
+      expect(partialIntents).to.have.lengthOf(1);
+
+      const action = await rebalanceActionStore.get(
+        'movement-undefined-status',
+      );
+      expect(action?.status).to.equal('failed');
+    });
+
+    it('does not fail long-running pending movement', async () => {
+      await rebalanceIntentStore.save({
+        id: 'intent-pending-movement',
+        status: 'in_progress',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n,
+        executionMethod: 'inventory',
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      await rebalanceActionStore.save({
+        id: 'movement-pending-old',
+        type: 'inventory_movement',
+        status: 'in_progress',
+        lastBridgeStatus: 'pending',
+        intentId: 'intent-pending-movement',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n,
+        createdAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+        updatedAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+      });
+
+      const partialIntents =
+        await tracker.getPartiallyFulfilledInventoryIntents();
+      expect(partialIntents).to.have.lengthOf(0);
+
+      const action = await rebalanceActionStore.get('movement-pending-old');
+      expect(action?.status).to.equal('in_progress');
+    });
+
+    it('does not fail old movement that recently became non-pending', async () => {
+      await rebalanceIntentStore.save({
+        id: 'intent-recent-not-found',
+        status: 'in_progress',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n,
+        executionMethod: 'inventory',
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      // Old by createdAt but only recently transitioned to not_found
+      await rebalanceActionStore.save({
+        id: 'movement-recent-not-found',
+        type: 'inventory_movement',
+        status: 'in_progress',
+        lastBridgeStatus: 'not_found',
+        nonPendingSince: Date.now() - 60_000, // only 1 min ago
+        intentId: 'intent-recent-not-found',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n,
+        createdAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+        updatedAt: Date.now(),
+      });
+
+      const partialIntents =
+        await tracker.getPartiallyFulfilledInventoryIntents();
+      expect(partialIntents).to.have.lengthOf(0);
+
+      const action = await rebalanceActionStore.get(
+        'movement-recent-not-found',
+      );
+      expect(action?.status).to.equal('in_progress');
+    });
+
+    it('handles mix of recent and stale movements', async () => {
+      await rebalanceIntentStore.save({
+        id: 'intent-mixed-movements',
+        status: 'in_progress',
+        origin: 1,
+        destination: 2,
+        amount: 1000000000000000000n,
+        executionMethod: 'inventory',
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      // One stale movement (non-pending for > 30 min)
+      await rebalanceActionStore.save({
+        id: 'movement-old',
+        type: 'inventory_movement',
+        status: 'in_progress',
+        lastBridgeStatus: 'not_found',
+        nonPendingSince: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+        intentId: 'intent-mixed-movements',
+        origin: 1,
+        destination: 2,
+        amount: 500000000000000000n,
+        createdAt: Date.now() - DEFAULT_MOVEMENT_STALENESS_MS - 1,
+        updatedAt: Date.now(),
+      });
+
+      // One recent movement
+      await rebalanceActionStore.save({
+        id: 'movement-new',
+        type: 'inventory_movement',
+        status: 'in_progress',
+        lastBridgeStatus: 'pending',
+        intentId: 'intent-mixed-movements',
+        origin: 1,
+        destination: 2,
+        amount: 500000000000000000n,
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      });
+
+      // Should skip because there's still a recent movement
+      const partialIntents =
+        await tracker.getPartiallyFulfilledInventoryIntents();
+      expect(partialIntents).to.have.lengthOf(0);
+
+      const staleAction = await rebalanceActionStore.get('movement-old');
+      expect(staleAction?.status).to.equal('in_progress');
+    });
   });
 
   describe('createRebalanceIntent', () => {

package/src/tracking/ActionTracker.ts

@@ -5,6 +5,7 @@ import type { MultiProtocolCore } from '@hyperlane-xyz/sdk';
 import type { Address, Domain } from '@hyperlane-xyz/utils';
 import { assert, parseWarpRouteMessage } from '@hyperlane-xyz/utils';
 
+import { DEFAULT_MOVEMENT_STALENESS_MS } from '../config/types.js';
 import type { ExternalBridgeRegistry } from '../interfaces/IExternalBridge.js';
 import type { ConfirmedBlockTags } from '../interfaces/IMonitor.js';
 import type {
@@ -589,12 +590,29 @@ export class ActionTracker implements IActionTracker {
       const actions = await this.getActionsForIntent(intent.id);
 
       // Check for in-flight inventory_movement actions
-      // Skip intents that have a bridge in progress - wait for it to complete
-      const hasInflightMovement = actions.some(
+      // Skip intents with active bridge movement(s). Movements still `pending` on
+      // the bridge are kept alive regardless of age. Non-pending movements are only
+      // failed after they've been in a non-pending state for the staleness window
+      // (nonPendingSince), preventing premature failure from transient not_found polls.
+      // `undefined` status (pre-deploy data) falls back to createdAt for staleness.
+      const inflightMovements = actions.filter(
         (a) => a.status === 'in_progress' && a.type === 'inventory_movement',
       );
+      const now = Date.now();
+      const staleMovements = inflightMovements.filter((a) => {
+        if (a.lastBridgeStatus === 'pending') return false;
+        // Use nonPendingSince when available; fall back to createdAt for
+        // pre-deploy data that lacks the field.
+        const nonPendingStart = a.nonPendingSince ?? a.createdAt;
+        return now - nonPendingStart >= DEFAULT_MOVEMENT_STALENESS_MS;
+      });
+      const staleMovementIds = new Set(staleMovements.map((a) => a.id));
+
+      const hasBlockingInflightMovement = inflightMovements.some(
+        (a) => !staleMovementIds.has(a.id),
+      );
 
-      if (hasInflightMovement) {
+      if (hasBlockingInflightMovement) {
         this.logger.debug(
           { intentId: intent.id },
           'Skipping partial intent - has in-flight inventory movement',
@@ -602,6 +620,22 @@ export class ActionTracker implements IActionTracker {
         continue;
       }
 
+      // Fail stale movements so the intent can proceed
+      for (const movement of staleMovements) {
+        await this.failRebalanceAction(movement.id);
+        this.logger.warn(
+          {
+            actionId: movement.id,
+            age: now - movement.createdAt,
+            nonPendingDuration:
+              now - (movement.nonPendingSince ?? movement.createdAt),
+            intentId: intent.id,
+            lastBridgeStatus: movement.lastBridgeStatus,
+          },
+          'Failing stale inventory movement to unblock intent',
+        );
+      }
+
       // Compute amounts from action states
       const completedAmount = actions
         .filter(
@@ -716,6 +750,10 @@ export class ActionTracker implements IActionTracker {
             'Inventory movement failed',
           );
         } else if (status.status === 'pending') {
+          await this.rebalanceActionStore.update(action.id, {
+            lastBridgeStatus: 'pending',
+            nonPendingSince: undefined,
+          });
           this.logger.debug(
             {
               actionId: action.id,
@@ -724,8 +762,19 @@ export class ActionTracker implements IActionTracker {
             },
             'Inventory movement still pending',
           );
+        } else if (status.status === 'not_found') {
+          await this.rebalanceActionStore.update(action.id, {
+            lastBridgeStatus: 'not_found',
+            nonPendingSince: action.nonPendingSince ?? Date.now(),
+          });
+          this.logger.debug(
+            {
+              actionId: action.id,
+              txHash: action.txHash,
+            },
+            'Inventory movement not found',
+          );
         }
-        // status === 'not_found' - wait for next cycle
       } catch (error) {
         this.logger.debug(
           {

package/src/tracking/types.ts

@@ -1,6 +1,7 @@
 import type { Address, Domain } from '@hyperlane-xyz/utils';
 
 import type { ExternalBridgeType } from '../config/types.js';
+import type { BridgeTransferStatus } from '../interfaces/IExternalBridge.js';
 
 import type { IStore } from './store/IStore.js';
 
@@ -84,6 +85,8 @@ export interface RebalanceAction extends TrackedActionBase {
   // Fields for inventory_movement (external bridge)
   externalBridgeTransferId?: string; // External bridge transfer ID (e.g., LiFi transfer ID)
   externalBridgeId?: ExternalBridgeType; // External bridge identifier (e.g., 'lifi')
+  lastBridgeStatus?: BridgeTransferStatus['status']; // Last observed external bridge status
+  nonPendingSince?: number; // Timestamp when bridge status first became non-pending
 }
 
 // === Type Aliases for Stores ===