@humanode-hyperlane/rebalancer 27.2.2 → 1029.1.1

package/dist/tracking/ActionTracker.test.js

@@ -1,10 +1,19 @@
 import chai, { expect } from 'chai';
 import chaiAsPromised from 'chai-as-promised';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
 import { pino } from 'pino';
 import Sinon from 'sinon';
 import { EthJsonRpcBlockParameterTag } from '@humanode-hyperlane/sdk';
+import { ProtocolType, assert } from '@humanode-hyperlane/utils';
 import { DEFAULT_INTENT_TTL_MS, DEFAULT_MOVEMENT_STALENESS_MS, ExternalBridgeType, } from '../config/types.js';
-import { ActionTracker } from './ActionTracker.js';
+import { ID_A, ID_B, ID_C, buildMessage, } from '../test/pendingTransferMiddlewareMocks.js';
+import { ExplorerClient, } from '../utils/ExplorerClient.js';
+import { ActionTracker, TRANSFER_SYNC_ROW_LIMIT_MESSAGE, } from './ActionTracker.js';
+import { ExplorerRowLimitError } from './pendingTransferMiddlewares/PendingTransferMiddleware.js';
+import { RelayerDenylistMiddleware } from './pendingTransferMiddlewares/RelayerDenylistMiddleware.js';
 import { InMemoryStore } from './store/InMemoryStore.js';
 chai.use(chaiAsPromised);
 const testLogger = pino({ level: 'silent' });
@@ -270,6 +279,29 @@ describe('ActionTracker', () => {
             const transfers = await transferStore.getAll();
             expect(transfers).to.have.lengthOf(1);
         });
+        const syncFailures = [
+            {
+                name: 'logs an Explorer row limit failure as an error and rethrows it',
+                error: new ExplorerRowLimitError('row limit', []),
+                loggedAsError: true,
+            },
+            {
+                name: 'rethrows other Explorer failures without an error log',
+                error: new Error('explorer down'),
+                loggedAsError: false,
+            },
+        ];
+        for (const c of syncFailures) {
+            it(c.name, async () => {
+                const logger = pino({ level: 'silent' });
+                const error = Sinon.spy(logger, 'error');
+                explorerClient.getInflightUserTransfers.rejects(c.error);
+                const tracker = new ActionTracker(transferStore, rebalanceIntentStore, rebalanceActionStore, explorerClient, core, config, logger);
+                await expect(tracker.syncTransfers()).to.be.rejectedWith(c.error);
+                expect(error.calledWith(Sinon.match({ error: c.error.message }), TRANSFER_SYNC_ROW_LIMIT_MESSAGE)).to.equal(c.loggedAsError);
+                expect(error.callCount).to.equal(c.loggedAsError ? 1 : 0);
+            });
+        }
         it('should mark transfers as complete when delivered', async () => {
             // Pre-create transfer
             await transferStore.save({
@@ -290,6 +322,88 @@ describe('ActionTracker', () => {
             const transfer = await transferStore.get('0xmsg1');
             expect(transfer?.status).to.equal('complete');
         });
+        describe('with an ignored-transfer predicate', () => {
+            const trackedTransfer = {
+                id: '0xmsg1',
+                status: 'in_progress',
+                messageId: '0xmsg1',
+                origin: 1,
+                destination: 2,
+                amount: 100n,
+                sender: '0xuser1',
+                recipient: '0xuser2',
+                createdAt: 1_000,
+                updatedAt: 1_000,
+            };
+            function createTracker(isIgnoredTransfer, logger = testLogger) {
+                return new ActionTracker(transferStore, rebalanceIntentStore, rebalanceActionStore, explorerClient, core, {
+                    routersByDomain: config.routersByDomain,
+                    isIgnoredTransfer,
+                    bridges: config.bridges,
+                    rebalancerAddress: config.rebalancerAddress,
+                    intentTTL: config.intentTTL,
+                }, logger);
+            }
+            it('keeps an ignored tracked transfer stored but excludes it from in-progress transfers without checking delivery', async () => {
+                await transferStore.save(trackedTransfer);
+                const tracker = createTracker((id) => id === '0xmsg1');
+                await tracker.syncTransfers();
+                const transfer = await tracker.getTransfer('0xmsg1');
+                expect(transfer?.status).to.equal('in_progress');
+                expect(await tracker.getInProgressTransfers()).to.deep.equal([]);
+                expect(mailboxStub.isDelivered.called).to.be.false;
+            });
+            it('reserves an ignored transfer again once it is no longer ignored, even when absent from the Explorer result', async () => {
+                await transferStore.save(trackedTransfer);
+                let ignored = true;
+                const tracker = createTracker((id) => ignored && id === '0xmsg1');
+                explorerClient.getInflightUserTransfers.resolves([]);
+                await tracker.syncTransfers();
+                expect(await tracker.getInProgressTransfers()).to.deep.equal([]);
+                ignored = false;
+                await tracker.syncTransfers();
+                const inProgress = await tracker.getInProgressTransfers();
+                expect(inProgress.map((t) => [t.id, t.amount])).to.deep.equal([
+                    ['0xmsg1', 100n],
+                ]);
+            });
+            it('completes an ignored transfer on the first sync after it is no longer ignored once delivered', async () => {
+                await transferStore.save(trackedTransfer);
+                let ignored = true;
+                const tracker = createTracker((id) => ignored && id === '0xmsg1');
+                mailboxStub.isDelivered.resolves(true);
+                await tracker.syncTransfers();
+                expect(mailboxStub.isDelivered.called).to.be.false;
+                expect((await tracker.getTransfer('0xmsg1'))?.status).to.equal('in_progress');
+                ignored = false;
+                await tracker.syncTransfers();
+                expect(mailboxStub.isDelivered.calledOnce).to.be.true;
+                expect((await tracker.getTransfer('0xmsg1'))?.status).to.equal('complete');
+            });
+            it('keeps a tracked transfer the predicate does not ignore', async () => {
+                await transferStore.save(trackedTransfer);
+                await createTracker((id) => id === '0xother').syncTransfers();
+                const transfer = await transferStore.get('0xmsg1');
+                expect(transfer?.status).to.equal('in_progress');
+                expect(mailboxStub.isDelivered.calledOnce).to.be.true;
+            });
+            it('counts an ignored transfer on every sync while it stays ignored', async () => {
+                await transferStore.save(trackedTransfer);
+                const logger = pino({ level: 'silent' });
+                const info = Sinon.spy(logger, 'info');
+                const tracker = createTracker((id) => id === '0xmsg1', logger);
+                await tracker.syncTransfers();
+                await tracker.syncTransfers();
+                const syncedCounts = info
+                    .getCalls()
+                    .filter((call) => call.args.includes('Transfers synced'))
+                    .map((call) => call.args[0]);
+                expect(syncedCounts).to.deep.equal([
+                    { newTransfers: 0, completed: 0, ignored: 1, inProgress: 0 },
+                    { newTransfers: 0, completed: 0, ignored: 1, inProgress: 0 },
+                ]);
+            });
+        });
     });
     describe('syncRebalanceIntents', () => {
         it('should mark intents as complete when fully fulfilled', async () => {
@@ -450,6 +564,168 @@ describe('ActionTracker', () => {
             expect(updatedAction?.status).to.equal('in_progress');
         });
     });
+    describe('with a relayer denylist middleware', () => {
+        let dir;
+        let denylistPath;
+        beforeEach(() => {
+            dir = mkdtempSync(join(tmpdir(), 'action-tracker-denylist-'));
+            denylistPath = join(dir, 'denylist.json');
+            writeDenylist([]);
+            explorerClient.getInflightRebalanceActions.resolves([
+                buildMessage(ID_A),
+                buildMessage(ID_B),
+                buildMessage(ID_C),
+            ]);
+        });
+        afterEach(() => {
+            rmSync(dir, { recursive: true, force: true });
+        });
+        function writeDenylist(messageIds, matches = []) {
+            writeFileSync(denylistPath, JSON.stringify({
+                rules: [
+                    ...messageIds.map((messageId) => ({ messageId })),
+                    ...matches,
+                ].map((match) => ({ reason: 'settled', addedBy: 'tester', match })),
+            }));
+        }
+        async function createDenylistTracker() {
+            const middleware = await RelayerDenylistMiddleware.create(explorerClient, { url: pathToFileURL(denylistPath).href, refreshIntervalMs: 0 }, testLogger);
+            return new ActionTracker(transferStore, rebalanceIntentStore, rebalanceActionStore, middleware, core, {
+                routersByDomain: config.routersByDomain,
+                isIgnoredTransfer: (id) => middleware.isIgnored(id),
+                isIgnoredRebalanceAction: (id) => middleware.isIgnoredRebalanceAction(id),
+                bridges: config.bridges,
+                rebalancerAddress: config.rebalancerAddress,
+                intentTTL: config.intentTTL,
+            }, testLogger);
+        }
+        async function activeIntentMessageIds(tracker) {
+            const actions = await rebalanceActionStore.getAll();
+            const intents = await tracker.getActiveRebalanceIntents();
+            return intents.map((intent) => {
+                const action = actions.find((a) => a.intentId === intent.id);
+                assert(action?.messageId, `Missing action for intent ${intent.id}`);
+                return action.messageId;
+            });
+        }
+        it('does not recover denylisted rebalance messages at startup', async () => {
+            writeDenylist([ID_A, ID_B]);
+            const tracker = await createDenylistTracker();
+            await tracker.initialize();
+            const actions = await rebalanceActionStore.getAll();
+            expect(actions.map((a) => a.id)).to.deep.equal([ID_C]);
+            const intents = await rebalanceIntentStore.getAll();
+            expect(intents.map((i) => [i.status, i.amount])).to.deep.equal([
+                ['in_progress', 100n],
+            ]);
+            expect(await activeIntentMessageIds(tracker)).to.deep.equal([ID_C]);
+        });
+        it('stops counting a tracked rebalance message once it is denylisted without changing its status', async () => {
+            const tracker = await createDenylistTracker();
+            await tracker.initialize();
+            expect(await activeIntentMessageIds(tracker)).to.have.members([
+                ID_A,
+                ID_B,
+                ID_C,
+            ]);
+            writeDenylist([ID_A]);
+            mailboxStub.isDelivered.resetHistory();
+            await tracker.syncRebalanceActions();
+            expect(await activeIntentMessageIds(tracker)).to.have.members([
+                ID_B,
+                ID_C,
+            ]);
+            const ignoredAction = await rebalanceActionStore.get(ID_A);
+            assert(ignoredAction, 'Expected the ignored action to stay stored');
+            expect(ignoredAction.status).to.equal('in_progress');
+            expect((await rebalanceIntentStore.get(ignoredAction.intentId))?.status).to.equal('in_progress');
+            expect(mailboxStub.isDelivered.calledWith(ID_A)).to.be.true;
+        });
+        it('completes a denylisted rebalance message the relayer still delivers', async () => {
+            const tracker = await createDenylistTracker();
+            await tracker.initialize();
+            writeDenylist([ID_A]);
+            mailboxStub.isDelivered.callsFake(async (id) => id === ID_A);
+            await tracker.syncRebalanceActions();
+            const action = await rebalanceActionStore.get(ID_A);
+            assert(action, 'Expected the delivered action to stay stored');
+            expect(action.status).to.equal('complete');
+            expect((await rebalanceIntentStore.get(action.intentId))?.status).to.equal('complete');
+        });
+        it('still fails a denylisted rebalance message past the intent TTL', async () => {
+            await rebalanceIntentStore.save({
+                id: 'expired-intent',
+                status: 'in_progress',
+                origin: 42161,
+                destination: 1,
+                amount: 100n,
+                createdAt: 0,
+                updatedAt: 0,
+            });
+            await rebalanceActionStore.save({
+                id: ID_A,
+                type: 'rebalance_message',
+                status: 'in_progress',
+                intentId: 'expired-intent',
+                messageId: ID_A,
+                origin: 42161,
+                destination: 1,
+                amount: 100n,
+                createdAt: 0,
+                updatedAt: 0,
+            });
+            writeDenylist([ID_A]);
+            const tracker = await createDenylistTracker();
+            await tracker.syncRebalanceActions();
+            expect(await activeIntentMessageIds(tracker)).to.have.members([
+                ID_B,
+                ID_C,
+            ]);
+            await tracker.syncRebalanceIntents();
+            expect((await rebalanceActionStore.get(ID_A))?.status).to.equal('failed');
+            expect((await rebalanceIntentStore.get('expired-intent'))?.status).to.equal('failed');
+        });
+        it('keeps ignoring a denylisted rebalance message pushed off the Explorer page', async () => {
+            const tracker = await createDenylistTracker();
+            await tracker.initialize();
+            writeDenylist([ID_A]);
+            await tracker.syncRebalanceActions();
+            expect(await activeIntentMessageIds(tracker)).to.not.include(ID_A);
+            const newerIds = Array.from({ length: 100 }, (_, i) => `0x${(i + 1).toString(16).padStart(64, '0')}`);
+            explorerClient.getInflightRebalanceActions.resolves(newerIds.map((id) => buildMessage(id)));
+            await tracker.syncRebalanceActions();
+            const countedIds = await activeIntentMessageIds(tracker);
+            expect(countedIds).to.not.include(ID_A);
+            expect(countedIds).to.include.members([ID_B, ID_C, ...newerIds]);
+            writeDenylist([]);
+            await tracker.syncRebalanceActions();
+            expect(await activeIntentMessageIds(tracker)).to.include(ID_A);
+        });
+        it('keeps recovering and counting rebalance messages a broad rule denies', async () => {
+            writeDenylist([], [{ destinationDomain: 1 }, { originDomain: 42161 }]);
+            const tracker = await createDenylistTracker();
+            await tracker.initialize();
+            await tracker.syncRebalanceActions();
+            expect(await activeIntentMessageIds(tracker)).to.have.members([
+                ID_A,
+                ID_B,
+                ID_C,
+            ]);
+        });
+        it('counts a rebalance message again once its rule is removed', async () => {
+            writeDenylist([ID_A, ID_B]);
+            const tracker = await createDenylistTracker();
+            await tracker.initialize();
+            expect(await activeIntentMessageIds(tracker)).to.deep.equal([ID_C]);
+            writeDenylist([]);
+            await tracker.syncRebalanceActions();
+            expect(await activeIntentMessageIds(tracker)).to.have.members([
+                ID_A,
+                ID_B,
+                ID_C,
+            ]);
+        });
+    });
     describe('syncInventoryMovementActions', () => {
         it('stores pending status on in-progress movement', async () => {
             await rebalanceActionStore.save({
@@ -611,6 +887,67 @@ describe('ActionTracker', () => {
             expect(result).to.have.lengthOf(1);
             expect(result[0].id).to.equal('intent-2');
         });
+        describe('with an ignored-rebalance-action predicate', () => {
+            const ignoredIds = new Set(['0xignored1', '0xignored2']);
+            function createTracker(isIgnoredRebalanceAction) {
+                return new ActionTracker(transferStore, rebalanceIntentStore, rebalanceActionStore, explorerClient, core, {
+                    routersByDomain: config.routersByDomain,
+                    isIgnoredRebalanceAction,
+                    bridges: config.bridges,
+                    rebalancerAddress: config.rebalancerAddress,
+                    intentTTL: config.intentTTL,
+                }, testLogger);
+            }
+            async function saveIntent(id, executionMethod) {
+                await rebalanceIntentStore.save({
+                    id,
+                    status: 'in_progress',
+                    origin: 1,
+                    destination: 2,
+                    amount: 300n,
+                    executionMethod,
+                    createdAt: Date.now(),
+                    updatedAt: Date.now(),
+                });
+            }
+            async function saveAction(id, intentId, type, status) {
+                await rebalanceActionStore.save({
+                    id,
+                    type,
+                    status,
+                    intentId,
+                    messageId: id,
+                    origin: 1,
+                    destination: 2,
+                    amount: 100n,
+                    createdAt: Date.now(),
+                    updatedAt: Date.now(),
+                });
+            }
+            it('drops only intents whose in-progress messages are all ignored', async () => {
+                await saveIntent('all-ignored', 'movable_collateral');
+                await saveAction('0xignored1', 'all-ignored', 'rebalance_message', 'in_progress');
+                await saveAction('0xdone', 'all-ignored', 'rebalance_message', 'complete');
+                await saveIntent('mixed');
+                await saveAction('0xignored2', 'mixed', 'rebalance_message', 'in_progress');
+                await saveAction('0xkept', 'mixed', 'rebalance_message', 'in_progress');
+                await saveIntent('no-actions', 'movable_collateral');
+                const result = await createTracker((id) => ignoredIds.has(id)).getActiveRebalanceIntents();
+                expect(result.map((i) => [i.id, i.amount])).to.have.deep.members([
+                    ['mixed', 300n],
+                    ['no-actions', 300n],
+                ]);
+            });
+            it('leaves inventory intents and their retry state untouched', async () => {
+                await saveIntent('inventory', 'inventory');
+                await saveAction('0xignored1', 'inventory', 'inventory_deposit', 'in_progress');
+                const withoutPredicate = createTracker();
+                const withPredicate = createTracker((id) => ignoredIds.has(id));
+                expect((await withPredicate.getActiveRebalanceIntents()).map((i) => i.id)).to.deep.equal(['inventory']);
+                expect(await withPredicate.getPartiallyFulfilledInventoryIntents()).to.deep.equal(await withoutPredicate.getPartiallyFulfilledInventoryIntents());
+                expect(await withPredicate.getPartiallyFulfilledInventoryIntents()).to.have.lengthOf(1);
+            });
+        });
     });
     describe('getPartiallyFulfilledInventoryIntents', () => {
         it('returns not_started inventory intents', async () => {
@@ -662,6 +999,45 @@ describe('ActionTracker', () => {
             expect(partialIntents[0].completedAmount).to.equal(400000000000000000n);
             expect(partialIntents[0].remaining).to.equal(600000000000000000n); // 0.6 ETH remaining
         });
+        it('ignores inventory_movement amounts when computing remaining', async () => {
+            await rebalanceIntentStore.save({
+                id: 'partial-intent-with-movement',
+                status: 'in_progress',
+                origin: 1,
+                destination: 2,
+                amount: 1000000000000000000n,
+                executionMethod: 'inventory',
+                createdAt: Date.now(),
+                updatedAt: Date.now(),
+            });
+            await rebalanceActionStore.save({
+                id: 'action-deposit',
+                type: 'inventory_deposit',
+                status: 'complete',
+                intentId: 'partial-intent-with-movement',
+                origin: 1,
+                destination: 2,
+                amount: 400000000000000000n,
+                createdAt: Date.now(),
+                updatedAt: Date.now(),
+            });
+            await rebalanceActionStore.save({
+                id: 'action-movement',
+                type: 'inventory_movement',
+                status: 'complete',
+                intentId: 'partial-intent-with-movement',
+                origin: 3,
+                destination: 2,
+                amount: 9000000000000000000n,
+                createdAt: Date.now(),
+                updatedAt: Date.now(),
+            });
+            const partialIntents = await tracker.getPartiallyFulfilledInventoryIntents();
+            expect(partialIntents).to.have.lengthOf(1);
+            expect(partialIntents[0].intent.id).to.equal('partial-intent-with-movement');
+            expect(partialIntents[0].completedAmount).to.equal(400000000000000000n);
+            expect(partialIntents[0].remaining).to.equal(600000000000000000n);
+        });
         it('does not return non-inventory intents', async () => {
             // Create a not_started intent without executionMethod: 'inventory'
             await rebalanceIntentStore.save({
@@ -1230,6 +1606,47 @@ describe('ActionTracker', () => {
             const updatedAction = await rebalanceActionStore.get('action-1');
             expect(updatedAction?.status).to.equal('complete');
         });
+        it('checks tracked delivery with the real Explorer client and no bridges', async () => {
+            const intent = {
+                id: 'intent-1',
+                status: 'in_progress',
+                origin: 1,
+                destination: 2,
+                amount: 100n,
+                createdAt: Date.now(),
+                updatedAt: Date.now(),
+            };
+            const action = {
+                id: 'action-1',
+                type: 'inventory_deposit',
+                status: 'in_progress',
+                intentId: 'intent-1',
+                messageId: '0xmsg1',
+                origin: 1,
+                destination: 2,
+                amount: 100n,
+                createdAt: Date.now(),
+                updatedAt: Date.now(),
+            };
+            await rebalanceIntentStore.save(intent);
+            await rebalanceActionStore.save(action);
+            config.bridges = [];
+            const fetchStub = Sinon.stub(globalThis, 'fetch').rejects(new Error('unexpected HTTP'));
+            tracker = new ActionTracker(transferStore, rebalanceIntentStore, rebalanceActionStore, new ExplorerClient('https://explorer.test', () => ProtocolType.Ethereum), core, config, testLogger);
+            mailboxStub.isDelivered.resolves(true);
+            try {
+                await tracker.syncRebalanceActions();
+                expect(fetchStub.called).to.be.false;
+            }
+            finally {
+                fetchStub.restore();
+            }
+            expect(mailboxStub.isDelivered.calledOnce).to.be.true;
+            const call = mailboxStub.isDelivered.firstCall;
+            expect(call.args[0]).to.equal('0xmsg1');
+            const updatedAction = await rebalanceActionStore.get('action-1');
+            expect(updatedAction?.status).to.equal('complete');
+        });
         it('should keep transfer in_progress when not delivered', async () => {
             await transferStore.save({
                 id: '0xmsg1',