@subsquid/batch-processor 1.0.0-portal-api.721f49 → 1.0.0

package/src/test/processor.test.ts

@@ -0,0 +1,651 @@
+import {buildChain, forkAt, joinFork, type MockBlock} from '@subsquid/util-internal-testing'
+import {
+    ForkException,
+    type BlockBatch,
+    type BlockRef,
+    type BlockStream,
+    type DataSource,
+    type StreamRequest,
+} from '@subsquid/util-internal-data-source'
+import {describe, expect, it} from 'vitest'
+import type {
+    Database,
+    FinalDatabase,
+    FinalDatabaseState,
+    FinalTxInfo,
+    HashAndHeight,
+    HotDatabase,
+    HotDatabaseState,
+    HotTxInfo,
+} from '../database'
+import {Processor, type DataHandlerContext} from '../run'
+
+// Block shape the Processor consumes — just a {header: BlockRef} envelope.
+interface TestBlock {
+    header: BlockRef
+}
+
+function wrap(headers: MockBlock[]): TestBlock[] {
+    return headers.map((h) => ({header: {number: h.height, hash: h.hash}}))
+}
+
+// In-process DataSource that serves a single pre-built chain. Only
+// getFinalizedHead and getFinalizedStream are wired — good enough for the
+// "final database, linear progress" case the Processor needs to handle.
+class InProcessSource implements DataSource<TestBlock> {
+    constructor(private readonly chain: TestBlock[]) {}
+
+    async getHead(): Promise<BlockRef> {
+        return this.tipRef()
+    }
+
+    async getFinalizedHead(): Promise<BlockRef> {
+        return this.tipRef()
+    }
+
+    getFinalizedStream(req: StreamRequest): BlockStream<TestBlock> {
+        return this.makeStream(req)
+    }
+
+    getStream(req: StreamRequest): BlockStream<TestBlock> {
+        return this.makeStream(req)
+    }
+
+    private tipRef(): BlockRef {
+        if (this.chain.length === 0) return {number: -1, hash: '0x'}
+        const tip = this.chain[this.chain.length - 1].header
+        return {number: tip.number, hash: tip.hash}
+    }
+
+    private async *makeStream(req: StreamRequest): BlockStream<TestBlock> {
+        const blocks = this.chain.filter(
+            (b) => b.header.number >= req.from && (req.to == null || b.header.number <= req.to),
+        )
+        if (blocks.length === 0) return
+        const finalizedHead = this.tipRef()
+        const batch: BlockBatch<TestBlock> = {blocks, finalizedHead}
+        yield batch
+    }
+}
+
+interface RecorderStore {
+    blocks: TestBlock[]
+}
+
+interface RecordedTransact {
+    prevHead: HashAndHeight
+    nextHead: HashAndHeight
+    isOnTop: boolean
+    blocks: TestBlock[]
+}
+
+class RecordingFinalDatabase implements FinalDatabase<RecorderStore> {
+    supportsHotBlocks: false = false
+    state: FinalDatabaseState = {height: -1, hash: '0x'}
+    transacts: RecordedTransact[] = []
+
+    async connect(): Promise<FinalDatabaseState> {
+        return this.state
+    }
+
+    async transact(info: FinalTxInfo, cb: (store: RecorderStore) => Promise<unknown>): Promise<void> {
+        const store: RecorderStore = {blocks: []}
+        await cb(store)
+        this.transacts.push({
+            prevHead: info.prevHead,
+            nextHead: info.nextHead,
+            isOnTop: info.isOnTop,
+            blocks: store.blocks,
+        })
+        this.state = {height: info.nextHead.height, hash: info.nextHead.hash}
+    }
+}
+
+interface RecordedHotTransact {
+    info: HotTxInfo
+    blocks: TestBlock[]
+}
+
+class RecordingHotDatabase implements HotDatabase<RecorderStore> {
+    supportsHotBlocks: true = true
+    state: HotDatabaseState = {height: -1, hash: '0x', top: []}
+    transacts: RecordedTransact[] = []
+    hotTransacts: RecordedHotTransact[] = []
+
+    constructor(private readonly hotGroupSize = Number.POSITIVE_INFINITY) {}
+
+    async connect(): Promise<HotDatabaseState> {
+        return this.state
+    }
+
+    async transact(info: FinalTxInfo, cb: (store: RecorderStore) => Promise<unknown>): Promise<void> {
+        const store: RecorderStore = {blocks: []}
+        await cb(store)
+        this.transacts.push({
+            prevHead: info.prevHead,
+            nextHead: info.nextHead,
+            isOnTop: info.isOnTop,
+            blocks: store.blocks,
+        })
+        this.state = {height: info.nextHead.height, hash: info.nextHead.hash, top: []}
+    }
+
+    async transactHot2(
+        info: HotTxInfo,
+        cb: (store: RecorderStore, sliceBeg: number, sliceEnd: number) => Promise<unknown>,
+    ): Promise<void> {
+        const store: RecorderStore = {blocks: []}
+        for (let sliceBeg = 0; sliceBeg < info.newBlocks.length; sliceBeg += this.hotGroupSize) {
+            await cb(store, sliceBeg, Math.min(sliceBeg + this.hotGroupSize, info.newBlocks.length))
+        }
+        this.hotTransacts.push({info, blocks: store.blocks})
+    }
+}
+
+// Stateful DataSource used for fork scenarios. `plan` is a list of "steps",
+// each consumed by one getStream invocation. A step can either yield a batch
+// or throw a ForkException; the source cycles through them in order and
+// terminates cleanly once the plan is exhausted.
+type StreamStep = {kind: 'yield'; batch: BlockBatch<TestBlock>} | {kind: 'fork'; fork: ForkException}
+
+class ProgrammableSource implements DataSource<TestBlock> {
+    private readonly plan: StreamStep[][] = []
+
+    constructor(private readonly chain: TestBlock[]) {}
+
+    // Each entry is what ONE getStream call should do, in order.
+    addStreamCall(steps: StreamStep[]): this {
+        this.plan.push(steps)
+        return this
+    }
+
+    async getHead(): Promise<BlockRef> {
+        return this.tipRef()
+    }
+
+    async getFinalizedHead(): Promise<BlockRef> {
+        return {number: -1, hash: '0x'}
+    }
+
+    getFinalizedStream(req: StreamRequest): BlockStream<TestBlock> {
+        return this.nextStream(req)
+    }
+
+    getStream(_req: StreamRequest): BlockStream<TestBlock> {
+        return this.nextStream(_req)
+    }
+
+    private tipRef(): BlockRef {
+        if (this.chain.length === 0) return {number: -1, hash: '0x'}
+        const tip = this.chain[this.chain.length - 1].header
+        return {number: tip.number, hash: tip.hash}
+    }
+
+    private async *nextStream(_req: StreamRequest): BlockStream<TestBlock> {
+        const steps = this.plan.shift()
+        if (!steps) return
+        for (const step of steps) {
+            if (step.kind === 'yield') {
+                yield step.batch
+            } else {
+                throw step.fork
+            }
+        }
+    }
+}
+
+describe('Processor', () => {
+    it('drives linear progress from DataSource to FinalDatabase in one transact call', async () => {
+        const headers = buildChain({from: 0, to: 4}) // 5 blocks
+        const chain = wrap(headers)
+
+        const src = new InProcessSource(chain)
+        const db: Database<RecorderStore> = new RecordingFinalDatabase()
+
+        const processor = new Processor<TestBlock, RecorderStore>(
+            src,
+            db,
+            async (ctx: DataHandlerContext<TestBlock, RecorderStore>) => {
+                for (const block of ctx.blocks) {
+                    ctx.store.blocks.push(block)
+                }
+            },
+        )
+
+        await processor.run()
+
+        const recorder = db as RecordingFinalDatabase
+        expect(recorder.transacts).toHaveLength(1)
+        const only = recorder.transacts[0]
+        expect(only.prevHead).toMatchObject({height: -1, hash: '0x'})
+        expect(only.nextHead).toMatchObject({height: 4, hash: '0x4'})
+        expect(only.isOnTop).toBe(true)
+        expect(only.blocks.map((b) => b.header.number)).toEqual([0, 1, 2, 3, 4])
+        expect(recorder.state).toMatchObject({height: 4, hash: '0x4'})
+    })
+
+    it('rejects non-continuous block order inside a batch', async () => {
+        const chain: TestBlock[] = [
+            {header: {number: 0, hash: '0x0'}},
+            {header: {number: 2, hash: '0x2'}},
+            {header: {number: 1, hash: '0x1'}},
+        ]
+
+        const src = new InProcessSource(chain)
+        const db = new RecordingFinalDatabase()
+        const processor = new Processor<TestBlock, RecorderStore>(src, db, async () => {})
+
+        await expect(processor.run()).rejects.toThrow(/Data is not continuous/)
+        expect(db.transacts).toHaveLength(0)
+    })
+
+    it('drives linear progress through a HotDatabase via transactHot2', async () => {
+        // Same five blocks, but fed through a HotDatabase. BlockBatch carries
+        // no finalizedHead, so every block is treated as unfinalized and
+        // routed through transactHot2.
+        const chain = wrap(buildChain({from: 0, to: 4}))
+
+        const src = new ProgrammableSource(chain).addStreamCall([
+            {kind: 'yield', batch: {blocks: chain, finalizedHead: undefined}},
+        ])
+        const db = new RecordingHotDatabase()
+
+        const processor = new Processor<TestBlock, RecorderStore>(src, db, async (ctx) => {
+            for (const block of ctx.blocks) {
+                ctx.store.blocks.push(block)
+            }
+        })
+
+        await processor.run()
+
+        expect(db.hotTransacts).toHaveLength(1)
+        const only = db.hotTransacts[0]
+        expect(only.info.baseHead).toMatchObject({height: -1, hash: '0x'})
+        expect(only.info.finalizedHead).toMatchObject({height: -1, hash: '0x'})
+        expect(only.info.newBlocks.map((b) => ({height: b.height, hash: b.hash}))).toEqual([
+            {height: 0, hash: '0x0'},
+            {height: 1, hash: '0x1'},
+            {height: 2, hash: '0x2'},
+            {height: 3, hash: '0x3'},
+            {height: 4, hash: '0x4'},
+        ])
+        expect(only.blocks.map((b) => b.header.number)).toEqual([0, 1, 2, 3, 4])
+    })
+
+    it('builds hot state from transactHot2 slice tips only', async () => {
+        const chain = wrap(buildChain({from: 0, to: 4}))
+
+        const src = new ProgrammableSource(chain).addStreamCall([
+            {kind: 'yield', batch: {blocks: chain, finalizedHead: undefined}},
+        ])
+        const db = new RecordingHotDatabase(2)
+
+        const processor = new Processor<TestBlock, RecorderStore>(src, db, async (ctx) => {
+            for (const block of ctx.blocks) {
+                ctx.store.blocks.push(block)
+            }
+        })
+
+        await processor.run()
+
+        expect(db.hotTransacts).toHaveLength(1)
+        expect(db.hotTransacts[0].blocks.map((b) => b.header.number)).toEqual([0, 1, 2, 3, 4])
+        expect(
+            (processor as unknown as {state: {unfinalizedHeads: BlockRef[]}}).state.unfinalizedHeads
+        ).toEqual([
+            {number: 1, hash: '0x1'},
+            {number: 3, hash: '0x3'},
+            {number: 4, hash: '0x4'},
+        ])
+    })
+
+    it('rolls back to the nearest known sparse head', async () => {
+        // Current sparse state is [1, 2, 6]. The fork view reaches block 4
+        // through the same block 2, so Processor should rewind
+        // state.unfinalizedHeads to the nearest known common point [1, 2]
+        // and re-request the stream from parentHash = 0x2.
+        const forkException = new ForkException(4, '0x4-fork', [
+            {number: 1, hash: '0x1'},
+            {number: 2, hash: '0x2'},
+            {number: 4, hash: '0x4-fork'},
+        ])
+        const forkBlocks: TestBlock[] = [
+            {header: {number: 3, hash: '0x3-fork'}},
+            {header: {number: 4, hash: '0x4-fork'}},
+        ]
+
+        const src = new ProgrammableSource(forkBlocks)
+            .addStreamCall([{kind: 'fork', fork: forkException}])
+            .addStreamCall([{kind: 'yield', batch: {blocks: forkBlocks, finalizedHead: undefined}}])
+
+        const db = new RecordingHotDatabase()
+        db.state = {
+            height: -1,
+            hash: '0x',
+            top: [
+                {height: 1, hash: '0x1'},
+                {height: 2, hash: '0x2'},
+                {height: 6, hash: '0x6'},
+            ],
+        }
+
+        const processor = new Processor<TestBlock, RecorderStore>(src, db, async () => {})
+
+        await processor.run()
+
+        expect(db.hotTransacts).toHaveLength(1)
+        // baseHead at the nearest known sparse common block (height 2, hash 0x2).
+        expect(db.hotTransacts[0].info.baseHead).toMatchObject({height: 2, hash: '0x2'})
+        expect(db.hotTransacts[0].info.newBlocks.map(b => b.hash)).toEqual(['0x3-fork', '0x4-fork'])
+    })
+
+    it('handles a deep reorg (>10 blocks) via ForkException', async () => {
+        // Main 0..12 (13 blocks). Fork at height 3, length 9 → replaces
+        // blocks 4..12 with 4a..12a. ForkException.previousBlocks carries
+        // the source's new view so findRollbackIndex can locate the common
+        // prefix when sparse hot state has it; otherwise the processor is
+        // allowed to roll back deeper.
+        const mainHeaders = buildChain({from: 0, to: 12})
+        const main = wrap(mainHeaders)
+        const forkHeaders = forkAt(mainHeaders, {at: 3, length: 9, suffix: 'a'})
+        const replay = wrap(joinFork(mainHeaders, forkHeaders))
+
+        // previousBlocks tells Processor what the source now considers the
+        // already-served chain. Hash divergence at height 4 triggers rollback.
+        const forkException = new ForkException(13, '0x12a', [
+            {number: 0, hash: '0x0'},
+            {number: 1, hash: '0x1'},
+            {number: 2, hash: '0x2'},
+            {number: 3, hash: '0x3'},
+            {number: 4, hash: '0x4a'},
+            {number: 5, hash: '0x5a'},
+            {number: 6, hash: '0x6a'},
+            {number: 7, hash: '0x7a'},
+            {number: 8, hash: '0x8a'},
+            {number: 9, hash: '0x9a'},
+            {number: 10, hash: '0x10a'},
+            {number: 11, hash: '0x11a'},
+            {number: 12, hash: '0x12a'},
+        ])
+
+        const src = new ProgrammableSource(main)
+            .addStreamCall([
+                {kind: 'yield', batch: {blocks: main, finalizedHead: undefined}},
+                {kind: 'fork', fork: forkException},
+            ])
+            .addStreamCall([{kind: 'yield', batch: {blocks: replay, finalizedHead: undefined}}])
+
+        const db = new RecordingHotDatabase()
+
+        const processor = new Processor<TestBlock, RecorderStore>(src, db, async (ctx) => {
+            for (const block of ctx.blocks) {
+                ctx.store.blocks.push(block)
+            }
+        })
+
+        await processor.run()
+
+        expect(db.hotTransacts).toHaveLength(2)
+        const reorg = db.hotTransacts[1]
+        expect(reorg.info.baseHead).toMatchObject({height: -1, hash: '0x'})
+        expect(reorg.info.newBlocks.map((b) => b.hash)).toEqual(replay.map((b) => b.header.hash))
+    })
+
+    it('handles cascading reorgs: two sequential ForkExceptions', async () => {
+        // main:     0 — 1 — 2 — 3 — 4
+        // fork A:                 3 — 4a — 5a    (common ancestor at height 3)
+        // fork B:         1 — 2 — 3b — 4b — 5b — 6b — 7b (deeper — common at 1)
+        //
+        // Three getStream calls, three transactHot2 commits. With dense hot
+        // state each baseHead would sit progressively deeper than the previous;
+        // with sparse state, rollback can land on an earlier known slice head.
+        const mainHeaders = buildChain({from: 0, to: 4})
+        const main = wrap(mainHeaders)
+
+        const branchAHeaders = forkAt(mainHeaders, {at: 3, length: 2, suffix: 'a'}) // 4a, 5a
+        const afterAHeaders = joinFork(mainHeaders, branchAHeaders)
+        const replayA = wrap(afterAHeaders).slice(1)
+
+        const branchBHeaders = forkAt(afterAHeaders, {at: 1, length: 6, suffix: 'b'}) // 2b..7b
+        const replayB = wrap(joinFork(afterAHeaders, branchBHeaders)).slice(1)
+
+        // First ForkException: source realises its chain at heights 4..5 (or
+        // beyond) differs from what it served. The fork view here retains
+        // 0..3 common, then diverges at 4.
+        const forkA = new ForkException(6, '0x5a', [
+            {number: 0, hash: '0x0'},
+            {number: 1, hash: '0x1'},
+            {number: 2, hash: '0x2'},
+            {number: 3, hash: '0x3'},
+            {number: 4, hash: '0x4a'},
+            {number: 5, hash: '0x5a'},
+        ])
+
+        // Second ForkException: now the source's view has shifted deeper —
+        // diverges from the Processor's committed chain. With sparse hot
+        // state, previousBlocks can prove the finalized genesis base, but not
+        // every intermediate hot block.
+        const forkB = new ForkException(8, '0x7b', [
+            {number: 0, hash: '0x0'},
+            {number: 1, hash: '0x1'},
+            {number: 2, hash: '0x2b'},
+            {number: 3, hash: '0x3b'},
+            {number: 4, hash: '0x4b'},
+            {number: 5, hash: '0x5b'},
+            {number: 6, hash: '0x6b'},
+            {number: 7, hash: '0x7b'},
+        ])
+
+        // Pin finalizedHead at genesis in every batch so that ProcessorState's
+        // chain = [finalizedHead, ...unfinalizedHeads] always has a stable
+        // chain[0] to drop in chain.slice(1, ...). Without this anchor, the
+        // slice drops the first unfinalized block and state drifts after
+        // cascading reorgs.
+        const anchoredFinalized = {number: 0, hash: '0x0'}
+        const src = new ProgrammableSource(main)
+            .addStreamCall([
+                {kind: 'yield', batch: {blocks: main, finalizedHead: anchoredFinalized}},
+                {kind: 'fork', fork: forkA},
+            ])
+            .addStreamCall([
+                {kind: 'yield', batch: {blocks: replayA, finalizedHead: anchoredFinalized}},
+                {kind: 'fork', fork: forkB},
+            ])
+            .addStreamCall([{kind: 'yield', batch: {blocks: replayB, finalizedHead: anchoredFinalized}}])
+
+        const db = new RecordingHotDatabase()
+
+        const processor = new Processor<TestBlock, RecorderStore>(src, db, async () => {})
+
+        await processor.run()
+
+        expect(db.hotTransacts).toHaveLength(3)
+
+        // First commit: main chain from genesis.
+        expect(db.hotTransacts[0].info.baseHead).toMatchObject({height: -1, hash: '0x'})
+        expect(db.hotTransacts[0].info.newBlocks.map((b) => b.hash)).toEqual(['0x0', '0x1', '0x2', '0x3', '0x4'])
+
+        // Second commit: the real common ancestor is height 3, but sparse
+        // state only proves finalized genesis, so replay starts from block 1.
+        expect(db.hotTransacts[1].info.baseHead).toMatchObject({height: 0, hash: '0x0'})
+        expect(db.hotTransacts[1].info.newBlocks.map((b) => b.hash)).toEqual(replayA.map((b) => b.header.hash))
+
+        // Third commit: the real common ancestor is height 1, but the
+        // cascading sparse rollback again starts from finalized genesis.
+        expect(db.hotTransacts[2].info.baseHead).toMatchObject({height: 0, hash: '0x0'})
+        expect(db.hotTransacts[2].info.newBlocks.map((b) => b.hash)).toEqual(replayB.map((b) => b.header.hash))
+    })
+
+    it('forwards BlockBatch.finalizedHead into transactHot2.info.finalizedHead', async () => {
+        // Mixed finalized+hot batch: the source hands over blocks 0..4 with
+        // finalizedHead = {3, '0x3'}. With a HotDatabase, Processor passes
+        // the entire batch through transactHot2 and expects info.finalizedHead
+        // to reflect the source's finality view — persistence layer is the
+        // one that splits the writes at the boundary.
+        const chain = wrap(buildChain({from: 0, to: 4}))
+
+        const src = new ProgrammableSource(chain).addStreamCall([
+            {
+                kind: 'yield',
+                batch: {
+                    blocks: chain,
+                    finalizedHead: {number: 3, hash: '0x3'},
+                },
+            },
+        ])
+
+        const db = new RecordingHotDatabase()
+
+        const processor = new Processor<TestBlock, RecorderStore>(src, db, async () => {})
+
+        await processor.run()
+
+        expect(db.hotTransacts).toHaveLength(1)
+        const only = db.hotTransacts[0]
+        expect(only.info.finalizedHead).toMatchObject({height: 3, hash: '0x3'})
+        expect(only.info.baseHead).toMatchObject({height: -1, hash: '0x'})
+        expect(only.info.newBlocks.map((b) => b.hash)).toEqual(['0x0', '0x1', '0x2', '0x3', '0x4'])
+    })
+
+    it('propagates non-ForkException errors thrown by the handler', async () => {
+        // If the user handler throws, the error must surface out of
+        // Processor.run() rather than being swallowed. Processor's outer
+        // try/catch only re-enters the while(true) loop for ForkException;
+        // everything else propagates.
+        const chain = wrap(buildChain({from: 0, to: 2}))
+        const src = new ProgrammableSource(chain).addStreamCall([
+            {kind: 'yield', batch: {blocks: chain, finalizedHead: undefined}},
+        ])
+
+        const db = new RecordingHotDatabase()
+
+        const processor = new Processor<TestBlock, RecorderStore>(src, db, async () => {
+            throw new Error('simulated handler crash')
+        })
+
+        await expect(processor.run()).rejects.toThrow(/simulated handler crash/)
+    })
+
+    it('processes a finality-only batch (empty blocks, advanced finalizedHead)', async () => {
+        const chain = wrap(buildChain({from: 0, to: 2}))
+        const src = new ProgrammableSource(chain).addStreamCall([
+            // Initial batch with three blocks and no finality.
+            {kind: 'yield', batch: {blocks: chain, finalizedHead: undefined}},
+            // Finality-only update: no new blocks, finalizedHead moves forward.
+            {kind: 'yield', batch: {blocks: [], finalizedHead: {number: 2, hash: '0x2'}}},
+        ])
+
+        const db = new RecordingHotDatabase()
+
+        const processor = new Processor<TestBlock, RecorderStore>(src, db, async () => {})
+
+        await processor.run()
+
+        // The finality-only batch must still reach the database so hot rows
+        // can be promoted even when no new best block arrived.
+        expect(db.hotTransacts).toHaveLength(2)
+        expect(db.hotTransacts[1].info.finalizedHead).toMatchObject({height: 2, hash: '0x2'})
+        expect(db.hotTransacts[1].info.newBlocks).toEqual([])
+    })
+
+    it('raises "Unable to process fork" when ForkException.previousBlocks is disjoint from state', async () => {
+        // If the data source's ForkException carries previousBlocks that share
+        // no common prefix with the Processor's committed chain AND the
+        // Processor has any finalized state, handleFork throws rather than
+        // silently wiping everything. The guard lives in ProcessorState:
+        //     if (rollbackIndex === -1) {
+        //         if (this.finalizedHead != null) throw new Error('Unable to process fork')
+        //         this.unfinalizedHeads = []
+        //     }
+        //
+        // This test pins finalizedHead to genesis on the first batch so the
+        // "finalized state present" branch fires, then fires a ForkException
+        // whose previousBlocks don't overlap with the committed chain at all.
+        const mainHeaders = buildChain({from: 0, to: 4})
+        const main = wrap(mainHeaders)
+
+        // Disjoint: different hashes at the SAME heights as committed state.
+        const disjointFork = new ForkException(5, '0xdisjoint5', [
+            {number: 0, hash: '0xZERO'},
+            {number: 1, hash: '0xONE'},
+            {number: 2, hash: '0xTWO'},
+            {number: 3, hash: '0xTHREE'},
+            {number: 4, hash: '0xFOUR'},
+        ])
+
+        const src = new ProgrammableSource(main).addStreamCall([
+            {
+                kind: 'yield',
+                batch: {blocks: main, finalizedHead: {number: 0, hash: '0x0'}},
+            },
+            {kind: 'fork', fork: disjointFork},
+        ])
+
+        const db = new RecordingHotDatabase()
+        const processor = new Processor<TestBlock, RecorderStore>(src, db, async () => {})
+
+        await expect(processor.run()).rejects.toThrow(/Unable to process fork/)
+
+        // The first batch still committed before the fork arrived.
+        expect(db.hotTransacts).toHaveLength(1)
+    })
+
+    it('accepts a data source whose getFinalizedHead never advances past genesis', async () => {
+        // No X-Sqd-Finalized-Head-* signal anywhere in the stream. Every
+        // block reaches the database as unfinalized via transactHot2;
+        // Processor runs to completion without waiting for finality to
+        // advance.
+        const main = wrap(buildChain({from: 0, to: 3}))
+
+        const src = new ProgrammableSource(main).addStreamCall([
+            {kind: 'yield', batch: {blocks: main, finalizedHead: undefined}},
+        ])
+
+        const db = new RecordingHotDatabase()
+        const processor = new Processor<TestBlock, RecorderStore>(src, db, async () => {})
+
+        await processor.run()
+
+        expect(db.hotTransacts).toHaveLength(1)
+        // info.finalizedHead falls back to the batch tip when no finality is
+        // reported (see run.ts:209-213 — unfinalizedIndex<0 branch).
+        // All four blocks routed through transactHot2.
+        expect(db.hotTransacts[0].info.newBlocks.map((b) => b.height)).toEqual([0, 1, 2, 3])
+    })
+
+    // Remaining Stage 5 scenarios from the roadmap. Subsumed by other tests,
+    // belonging to a different stage, or requiring a real data-source
+    // integration beyond the in-process harness. Explicit todos so the gaps
+    // stay visible until addressed.
+
+    // — groupSize > 1 mid-group baseHead. Processor is oblivious to
+    // persistence-layer grouping; it just hands info to transactHot2. The
+    // groupSize bug lives in typeorm-store and is covered by
+    // fork-deep.test.ts.
+    it.todo('groupSize > 1 mid-group baseHead')
+
+    // Deep EVM reorg end-to-end is still unverified on this layer. The
+    // deep-reorg test above feeds a synthetic ForkException with a complete
+    // previousBlocks history, so it exercises findRollbackIndex in isolation
+    // but not the real chain-specific datasource behavior: ensureContinuity
+    // in EvmRpcDataSource decides what previousBlocks payload to actually
+    // throw, and any N-block reorg will fire one ForkException per fork-
+    // point mismatch. A true end-to-end test would wire a mock EVM RPC (not
+    // a ProgrammableSource) through EvmRpcDataSource and into Processor +
+    // TypeormDatabase, then assert the final state is correct and the
+    // processor survives the full reorg cycle without hitting the
+    // 'Unable to process fork' guard for cases where the fork-point IS in
+    // our chain. Needs the full integration harness — deferred.
+    it.todo('deep EVM reorg end-to-end: EvmRpcDataSource + Processor + TypeormDatabase')
+
+    // coldIngest over-fetch + empty split. These test the internals of
+    // util-internal-ingest-tools/cold.ts, below the Processor boundary.
+    it.todo('coldIngest over-fetch at stopOnHead boundary')
+    it.todo('coldIngest empty split + isHead moving-top boundary')
+
+    // Gapped top[] from typeorm-store → HotProcessor bridge. Needs
+    // a real HotProcessor wired on top of a real TypeormDatabase returning
+    // a gap-structured top[]; the in-process Processor mocks here don't
+    // exercise the bridge.
+    it.todo('gapped top[] from DB → HotProcessor initialisation')
+})

package/src/util.ts

@@ -1,4 +1,4 @@
-import {HashAndHeight} from './database'
+import type {BlockRef} from '@subsquid/util-internal-data-source'
 
 
 export function timeInterval(seconds: number): string {
@@ -29,8 +29,8 @@ export function getItemsCount(blocks: any[]): number {
 }
 
 
-export function formatHead(head: HashAndHeight): string {
-    return `${head.height}#${shortHash(head.hash)}`
+export function formatHead(head: BlockRef): string {
+    return `${head.number}#${shortHash(head.hash)}`
 }
 
 

package/lib/metrics.d.ts

@@ -1,21 +0,0 @@
-export declare class Metrics {
-    private chainHeight;
-    private lastBlock;
-    private mappingSpeed;
-    private mappingItemSpeed;
-    private blockProgress;
-    setChainHeight(height: number): void;
-    setLastProcessedBlock(height: number): void;
-    updateProgress(processed: number, left: number, time?: bigint): void;
-    registerBatch(batchSize: number, batchItemSize: number, batchMappingStartTime: bigint, batchMappingEndTime: bigint): void;
-    getChainHeight(): number;
-    getLastProcessedBlock(): number;
-    getSyncSpeed(): number;
-    getSyncEtaSeconds(): number;
-    getSyncRatio(): number;
-    getMappingSpeed(): number;
-    getMappingItemSpeed(): number;
-    getStatusLine(): string;
-    install(): void;
-}
-//# sourceMappingURL=metrics.d.ts.map