@subsquid/solana-normalization 0.0.3 → 1.0.0-portal-api.d887ad

package/src/mapping.test.ts

@@ -0,0 +1,76 @@
+import {GetBlock} from '@subsquid/solana-rpc-data'
+import {toJSON} from '@subsquid/util-internal-json'
+import {fixUnsafeIntegers} from '@subsquid/util-internal-json-fix-unsafe-integers'
+import assert, {fail} from 'assert'
+import * as fs from 'fs'
+import {it} from 'node:test'
+import * as Path from 'path'
+import {Journal, mapRpcBlock} from './mapping'
+import {removeVotes} from './votes'
+
+
+const FIXTURES_DIR = Path.resolve(__dirname, '../fixtures')
+
+
+interface Fixture {
+    name: string
+    block: GetBlock
+    result: any
+}
+
+
+function* listFixtures(): Iterable<Fixture> {
+    for (let name of fs.readdirSync(FIXTURES_DIR)) {
+        let block: GetBlock
+        let result: any
+        try {
+            block = JSON.parse(
+                fixUnsafeIntegers(fs.readFileSync(Path.join(FIXTURES_DIR, name, 'block.json'), 'utf-8'))
+            )
+            result = JSON.parse(
+                fs.readFileSync(Path.join(FIXTURES_DIR, name, 'result.json'), 'utf-8')
+            )
+        } catch(err: any) {
+            if (err.code === 'ENOENT' || err.code == 'ENOTDIR') {
+                continue
+            } else {
+                throw err
+            }
+        }
+        yield {name, block, result}
+    }
+}
+
+
+const failingJournal: Journal = {
+    warn(props: any, msg: string) {
+        throw new Error(`got warning: ${msg}, props: ${JSON.stringify(props)}`)
+    },
+    error(props: any, msg: string) {
+        throw new Error(`got error: ${msg}, props: ${JSON.stringify(props)}`)
+    }
+}
+
+
+for (let fix of listFixtures()) {
+    it(fix.name, () => {
+        let result = mapRpcBlock(0, fix.block, failingJournal)
+        removeVotes(result)
+        let resultJson = normalizeJson(result)
+        try {
+            assert.deepStrictEqual(resultJson, fix.result)
+        } catch(err: any) {
+            // diff, that comes from deepStrictEqual is not useful for large objects
+            fs.writeFileSync(
+                Path.join(FIXTURES_DIR, fix.name, 'result.temp.json'),
+                JSON.stringify(resultJson, null, 2)
+            )
+            fail('result is different from a reference')
+        }
+    })
+}
+
+
+function normalizeJson(obj: any): any {
+    return JSON.parse(JSON.stringify(toJSON(obj)))
+}

package/src/mapping.ts

@@ -1,38 +1,28 @@
 import type * as rpc from '@subsquid/solana-rpc-data'
-import type {Base58Bytes} from '@subsquid/solana-rpc-data'
-import {addErrorContext, unexpectedCase} from '@subsquid/util-internal'
+import {addErrorContext, assertNotNull} from '@subsquid/util-internal'
 import assert from 'assert'
 import {Balance, Block, BlockHeader, Instruction, LogMessage, Reward, TokenBalance, Transaction} from './data'
-import {InvokeMessage, InvokeResultMessage, LogTruncatedMessage, Message, parseLogMessage} from './log-parser'
+import {InstructionTreeTraversal, MessageStream, ParsingError} from './instruction-parser'
+import {LogTruncatedMessage} from './log-parser'
+import {Journal, TransactionContext} from './transaction-context'
 
 
-export function mapRpcBlock(src: rpc.Block): Block {
+export {Journal}
+
+
+export function mapRpcBlock(slot: number, src: rpc.GetBlock, journal: Journal): Block {
     let header: BlockHeader = {
-        hash: src.hash,
-        height: src.height,
-        slot: src.slot,
-        parentSlot: src.block.parentSlot,
-        parentHash: src.block.previousBlockhash,
-        timestamp: src.block.blockTime ?? 0
+        hash: src.blockhash,
+        height: assertNotNull(src.blockHeight, '.blockHeight is not available'),
+        number: slot,
+        parentNumber: src.parentSlot,
+        parentHash: src.previousBlockhash,
+        timestamp: src.blockTime ?? 0
     }
 
-    let instructions: Instruction[] = []
-    let logs: LogMessage[] = []
-    let balances: Balance[] = []
-    let tokenBalances: TokenBalance[] = []
-
-    let transactions = src.block.transactions
-        ?.map((tx, i) => {
-            try {
-                return mapRpcTransaction(i, tx, instructions, logs, balances, tokenBalances)
-            } catch(err: any) {
-                throw addErrorContext(err, {
-                    blockTransaction: tx.transaction.signatures[0]
-                })
-            }
-        }) ?? []
+    let items = new ItemMapping(journal, src)
 
-    let rewards = src.block.rewards?.map(s => {
+    let rewards = src.rewards?.map(s => {
         let reward: Reward = {
             pubkey: s.pubkey,
             lamports: BigInt(s.lamports),
@@ -52,393 +42,124 @@ export function mapRpcBlock(src: rpc.Block): Block {
 
     return {
         header,
-        transactions,
-        instructions,
-        logs,
-        balances,
-        tokenBalances,
+        transactions: items.transactions,
+        instructions: items.instructions,
+        logs: items.logs,
+        balances: items.balances,
+        tokenBalances: items.tokenBalances,
         rewards: rewards || []
     }
 }
 
 
-function mapRpcTransaction(
-    transactionIndex: number,
-    src: rpc.Transaction,
-    instructions: Instruction[],
-    logs: LogMessage[],
-    balances: Balance[],
-    tokenBalances: TokenBalance[]
-): Transaction {
-    let tx: Transaction = {
-        transactionIndex,
-        version: src.version,
-        accountKeys: src.transaction.message.accountKeys,
-        addressTableLookups: src.transaction.message.addressTableLookups ?? [],
-        numReadonlySignedAccounts: src.transaction.message.header.numReadonlySignedAccounts,
-        numReadonlyUnsignedAccounts: src.transaction.message.header.numReadonlyUnsignedAccounts,
-        numRequiredSignatures: src.transaction.message.header.numRequiredSignatures,
-        recentBlockhash: src.transaction.message.recentBlockhash,
-        signatures: src.transaction.signatures,
-        err: src.meta.err,
-        computeUnitsConsumed: BigInt(src.meta.computeUnitsConsumed ?? 0),
-        fee: BigInt(src.meta.fee),
-        loadedAddresses: src.meta.loadedAddresses ?? {readonly: [], writable: []},
-        hasDroppedLogMessages: false
-    }
-
-    let accounts: Base58Bytes[]
-    if (tx.version === 'legacy') {
-        accounts = tx.accountKeys
-    } else {
-        assert(src.meta?.loadedAddresses)
-        accounts = tx.accountKeys.concat(
-            src.meta.loadedAddresses.writable,
-            src.meta.loadedAddresses.readonly
-        )
-    }
-
-    let getAccount = (index: number): Base58Bytes => {
-        assert(index < accounts.length)
-        return accounts[index]
-    }
-
-    new InstructionParser(
-        getAccount,
-        tx,
-        src,
-        src.meta.logMessages?.map(parseLogMessage),
-        instructions,
-        logs
-    ).parse()
-
-    balances.push(
-        ...mapBalances(getAccount, transactionIndex, src)
-    )
-
-    tokenBalances.push(
-        ...mapTokenBalances(getAccount, transactionIndex, src)
-    )
-
-    return tx
-}
-
-
-const PROGRAMS_MISSING_INVOKE_LOG = new Set([
-    'AddressLookupTab1e1111111111111111111111111',
-    'BPFLoader1111111111111111111111111111111111',
-    'BPFLoader2111111111111111111111111111111111',
-    'BPFLoaderUpgradeab1e11111111111111111111111',
-    'Ed25519SigVerify111111111111111111111111111',
-    'KeccakSecp256k11111111111111111111111111111',
-    'NativeLoader1111111111111111111111111111111',
-    'ZkTokenProof1111111111111111111111111111111',
-])
-
-
-class InstructionParser {
-    private pos = 0
-    private messages: Message[]
-    private messagePos = 0
-    private messagesTruncated = false
-    private errorPos?: number
-    private lastAddress: number[] = []
+class ItemMapping {
+    transactions: Transaction[] = []
+    instructions: Instruction[] = []
+    logs: LogMessage[] = []
+    balances: Balance[] = []
+    tokenBalances: TokenBalance[] = []
 
     constructor(
-        private getAccount: (index: number) => Base58Bytes,
-        private tx: Transaction,
-        private src: rpc.Transaction,
-        messages: Message[] | undefined,
-        private instructions: Instruction[],
-        private logs: LogMessage[]
+        private journal: Journal,
+        block: rpc.GetBlock
     ) {
-        if (messages == null) {
-            this.messages = []
-            this.messagesTruncated = true
-        } else {
-            this.messages = messages
-        }
-        let err: any = this.src.meta.err
-        if (err) {
-            if ('InstructionError' in err) {
-                let pos = err['InstructionError'][0]
-                assert(typeof pos == 'number')
-                assert(0 <= pos && pos < this.src.transaction.message.instructions.length)
-                this.errorPos = pos
+        let transactions = block.transactions ?? []
+        for (let i = 0; i < transactions.length; i++) {
+            let tx = transactions[i]
+            try {
+                this.processTransaction(i, tx)
+            } catch(err: any) {
+                throw addErrorContext(err, {
+                    transactionHash: tx.transaction.signatures[0]
+                })
             }
         }
     }
 
-    parse(): void {
-        while (this.pos < this.src.transaction.message.instructions.length) {
-            let instruction = this.src.transaction.message.instructions[this.pos]
-
-            let inner = this.src.meta.innerInstructions
-                ?.filter(i => i.index === this.pos)
-                .flatMap(i => i.instructions) ?? []
-
-            if (this.errorPos == null || this.errorPos >= this.pos) {
-                let instructions = [instruction].concat(inner)
-                let end = this.traverse(instructions, 0, 1)
-                assert(end == instructions.length)
+    private processTransaction(transactionIndex: number, src: rpc.Transaction): void {
+        let mapped = mapTransaction(transactionIndex, src)
+
+        let ctx = new TransactionContext(transactionIndex, src, this.journal)
+
+        let messages = new MessageStream(src.meta.logMessages ?? [])
+
+        let insCheckPoint = this.instructions.length
+        let logCheckPoint = this.logs.length
+        try {
+            this.traverseInstructions(ctx, messages)
+            mapped.hasDroppedLogMessages = messages.truncated
+        } catch(err: any) {
+            if (err instanceof ParsingError) {
+                // report parsing problem
+                let {msg, ...props} = err
+                ctx.error(props, msg)
+                // reparse without log messages
+                mapped.hasDroppedLogMessages = true
+                // restore state before failed traversal
+                this.instructions = this.instructions.slice(0, insCheckPoint)
+                this.logs = this.logs.slice(0, logCheckPoint)
+                // traverse again with dummy truncated MessageStream
+                this.traverseInstructions(
+                    ctx,
+                    new MessageStream([new LogTruncatedMessage().toString()])
+                )
             } else {
-                this.assert(inner.length == 0, false, 0, 'seemingly non-executed instruction has inner instructions')
-                this.push(1, instruction)
-            }
-
-            this.pos += 1
-        }
-        this.tx.hasDroppedLogMessages = this.tx.hasDroppedLogMessages || this.messagesTruncated
-    }
-
-    private traverse(
-        instructions: rpc.Instruction[],
-        pos: number,
-        stackHeight: number
-    ): number {
-        this.assert(pos < instructions.length, true, 0, 'unexpected and of inner instructions')
-
-        let instruction = instructions[pos]
-
-        this.assert(
-            instruction.stackHeight == null || instruction.stackHeight == stackHeight,
-            false, pos, 'instruction has unexpected stack height'
-        )
-
-        let msg: Message | undefined
-        if (this.messagePos < this.messages.length) {
-            msg = this.messages[this.messagePos]
-        }
-
-        if (msg?.kind == 'truncate') {
-            this.messagePos += 1
-            this.messagesTruncated = true
-        }
-
-        if (this.messagesTruncated) {
-            this.push(stackHeight, instruction)
-            return this.logLessTraversal(stackHeight, instructions, pos + 1)
-        }
-
-        let programId = this.getAccount(instruction.programIdIndex)
-
-        if (msg?.kind === 'invoke' && msg.programId === programId) {
-            this.assert(msg.stackHeight == stackHeight, true, pos, 'invoke message has unexpected stack height')
-            this.messagePos += 1
-            return this.invokeInstruction(stackHeight, instructions, pos)
-        }
-
-        if (PROGRAMS_MISSING_INVOKE_LOG.has(programId)) {
-            let dropped = this.dropInvokeLessInstructionMessages(pos)
-            let ins = this.push(stackHeight, instruction)
-            ins.hasDroppedLogMessages = ins.hasDroppedLogMessages || dropped
-            this.tx.hasDroppedLogMessages = this.tx.hasDroppedLogMessages || dropped
-            return this.invokeLessTraversal(dropped, stackHeight, instructions, pos + 1)
-        }
-
-        // FIXME: add an option to ignore this
-        throw this.error(true, pos, 'missing invoke message')
-    }
-
-    private dropInvokeLessInstructionMessages(pos: number): boolean {
-        let initialPos = this.messagePos
-        while (this.messagePos < this.messages.length && !this.messagesTruncated) {
-            let msg = this.messages[this.messagePos]
-            switch(msg.kind) {
-                case 'log':
-                case 'data':
-                case 'cu':
-                case 'other':
-                    this.messagePos += 1
-                    break
-                case 'truncate':
-                    this.messagePos += 1
-                    this.messagesTruncated = true
-                    return true
-                case 'invoke':
-                    return this.messagePos - initialPos > 0
-                case 'invoke-result':
-                    throw this.error(true, pos, `invoke result message does not match any invoke`)
-                default:
-                    throw unexpectedCase()
-            }
-        }
-        return false
-    }
-
-    private invokeInstruction(
-        stackHeight: number,
-        instructions: rpc.Instruction[],
-        instructionPos: number
-    ): number {
-        let ins = this.push(stackHeight, instructions[instructionPos])
-        let pos = instructionPos + 1
-        while (true) {
-            let token = this.takeInstructionMessages(ins, instructionPos)
-            switch(token.kind) {
-                case 'invoke':
-                    pos = this.traverse(instructions, pos, stackHeight + 1)
-                    break
-                case 'invoke-result':
-                    if (token.programId != ins.programId) {
-                        throw this.error(true, instructionPos,
-                            `invoke result message and instruction program ids don't match`
-                        )
-                    }
-                    if (token.error) {
-                        ins.error = token.error
-                    }
-                    pos = this.invokeLessTraversal(true, stackHeight, instructions, pos)
-                    this.messagePos += 1
-                    return pos
-                case 'truncate':
-                    ins.hasDroppedLogMessages = true
-                    return this.logLessTraversal(stackHeight, instructions, pos)
-                default:
-                    throw unexpectedCase()
+                throw err
             }
         }
-    }
-
-    private takeInstructionMessages(
-        ins: Instruction,
-        pos: number
-    ): InvokeMessage | InvokeResultMessage | LogTruncatedMessage {
-        if (this.messagesTruncated) return new LogTruncatedMessage()
-        while (this.messagePos < this.messages.length) {
-            let msg = this.messages[this.messagePos]
-            switch(msg.kind) {
-                case 'log':
-                case 'data':
-                case 'other':
-                    this.logs.push({
-                        transactionIndex: ins.transactionIndex,
-                        logIndex: this.messagePos,
-                        instructionAddress: ins.instructionAddress,
-                        programId: ins.programId,
-                        kind: msg.kind,
-                        message: msg.message
-                    })
-                    break
-                case 'cu':
-                    if (ins.programId != msg.programId) {
-                        throw this.error(true, pos, 'unexpected programId in compute unit message')
-                    }
-                    ins.computeUnitsConsumed = msg.consumed
-                    break
-                case 'invoke':
-                case 'invoke-result':
-                    return msg
-                case 'truncate':
-                    this.messagesTruncated = true
-                    this.messagePos += 1
-                    return msg
-                default:
-                    throw unexpectedCase()
-            }
-            this.messagePos += 1
-        }
-        throw this.error(false, pos, 'unexpected end of log messages')
-    }
-
-    private invokeLessTraversal(
-        messagesDropped: boolean,
-        parentStackHeight: number,
-        instructions: rpc.Instruction[],
-        pos: number
-    ): number {
-        return this.logLessTraversal(parentStackHeight, instructions, pos, (ins, pos) => {
-            ins.hasDroppedLogMessages = ins.hasDroppedLogMessages || messagesDropped
-            if (PROGRAMS_MISSING_INVOKE_LOG.has(ins.programId)) {
-            } else {
-                throw this.error(false, pos, 'invoke message is missing')
-            }
-        })
-    }
 
-    private logLessTraversal(
-        parentStackHeight: number,
-        instructions: rpc.Instruction[],
-        pos: number,
-        cb?: (ins: Instruction, pos: number) => void
-    ): number {
-        while (pos < instructions.length) {
-            let instruction = instructions[pos]
-            let stackHeight = instruction.stackHeight ?? 2
-            if (stackHeight > parentStackHeight) {
-                let ins = this.push(stackHeight, instruction)
-                cb?.(ins, pos)
-                pos += 1
-            } else {
-                return pos
-            }
-        }
-        return pos
+        this.transactions.push(mapped)
+        this.balances.push(...mapBalances(ctx))
+        this.tokenBalances.push(...mapTokenBalances(ctx))
     }
 
-    private push(stackHeight: number, src: rpc.Instruction): Instruction {
-        assert(stackHeight > 0)
+    private traverseInstructions(ctx: TransactionContext, messages: MessageStream): void {
+        for (let i = 0; i < ctx.tx.transaction.message.instructions.length; i++) {
+            let ins = ctx.tx.transaction.message.instructions[i]
 
-        if (src.stackHeight != null) {
-            assert(stackHeight === src.stackHeight)
-        }
-
-        let address = this.lastAddress.slice()
-
-        while (address.length > stackHeight) {
-            address.pop()
-        }
-
-        if (address.length === stackHeight) {
-            address[stackHeight - 1] += 1
-        } else {
-            assert(address.length + 1 == stackHeight)
-            address[stackHeight - 1] = 0
-        }
+            let inner = ctx.tx.meta.innerInstructions?.flatMap(pack => {
+                return pack.index === i ? pack.instructions : []
+            })
 
-        let i: Instruction = {
-            transactionIndex: this.tx.transactionIndex,
-            instructionAddress: address,
-            programId: this.getAccount(src.programIdIndex),
-            accounts: src.accounts.map(a => this.getAccount(a)),
-            data: src.data,
-            isCommitted: !this.tx.err,
-            hasDroppedLogMessages: this.messagesTruncated
+            new InstructionTreeTraversal(
+                ctx,
+                messages,
+                i,
+                ins,
+                inner ?? [],
+                this.instructions,
+                this.logs
+            )
         }
-
-        this.instructions.push(i)
-        this.lastAddress = address
-        return i
     }
+}
 
-    private assert(ok: unknown, withMessagePos: boolean, innerPos: number, msg: string): asserts ok {
-        if (!ok) throw this.error(withMessagePos, innerPos, msg)
-    }
 
-    private error(withMessagePos: boolean, innerPos: number, msg: string): Error {
-        let loc = `stopped at instruction ${this.pos}`
-        if (innerPos > 0) {
-            loc += `, inner instruction ${innerPos - 1})`
-        }
-        if (withMessagePos && this.messagePos < this.messages.length) {
-            loc += ` and log message ${this.messagePos}`
-        }
-        return new Error(
-            `Failed to process transaction ${this.tx.signatures[0]}: ${loc}: ${msg}`
-        )
+function mapTransaction(transactionIndex: number, src: rpc.Transaction): Transaction {
+    return {
+        transactionIndex,
+        version: src.version,
+        accountKeys: src.transaction.message.accountKeys,
+        addressTableLookups: src.transaction.message.addressTableLookups ?? [],
+        numReadonlySignedAccounts: src.transaction.message.header.numReadonlySignedAccounts,
+        numReadonlyUnsignedAccounts: src.transaction.message.header.numReadonlyUnsignedAccounts,
+        numRequiredSignatures: src.transaction.message.header.numRequiredSignatures,
+        recentBlockhash: src.transaction.message.recentBlockhash,
+        signatures: src.transaction.signatures,
+        err: src.meta.err,
+        computeUnitsConsumed: BigInt(src.meta.computeUnitsConsumed ?? 0),
+        fee: BigInt(src.meta.fee),
+        loadedAddresses: src.meta.loadedAddresses ?? {readonly: [], writable: []},
+        hasDroppedLogMessages: false
     }
 }
 
 
-function mapBalances(
-    getAccount: (idx: number) => Base58Bytes,
-    transactionIndex: number,
-    tx: rpc.Transaction,
-): Balance[] {
+function mapBalances(ctx: TransactionContext): Balance[] {
     let balances: Balance[] = []
 
-    let pre = tx.meta.preBalances
-    let post = tx.meta.postBalances
+    let pre = ctx.tx.meta.preBalances
+    let post = ctx.tx.meta.postBalances
 
     assert(pre.length == post.length)
 
@@ -447,8 +168,8 @@ function mapBalances(
             // nothing changed, don't create an entry
         } else {
             balances.push({
-                transactionIndex,
-                account: getAccount(i),
+                transactionIndex: ctx.transactionIndex,
+                account: ctx.getAccount(i),
                 pre: BigInt(pre[i]),
                 post: BigInt(post[i])
             })
@@ -465,26 +186,22 @@ function mapBalances(
 }
 
 
-function mapTokenBalances(
-    getAccount: (idx: number) => Base58Bytes,
-    transactionIndex: number,
-    tx: rpc.Transaction
-): TokenBalance[] {
+function mapTokenBalances(ctx: TransactionContext): TokenBalance[] {
     let balances: TokenBalance[] = []
 
     let preBalances = new Map(
-        tx.meta.preTokenBalances?.map(b => [getAccount(b.accountIndex), b])
+        ctx.tx.meta.preTokenBalances?.map(b => [ctx.getAccount(b.accountIndex), b])
     )
 
     let postBalances = new Map(
-        tx.meta.postTokenBalances?.map(b => [getAccount(b.accountIndex), b])
+        ctx.tx.meta.postTokenBalances?.map(b => [ctx.getAccount(b.accountIndex), b])
     )
 
     for (let [account, post] of postBalances.entries()) {
         let pre = preBalances.get(account)
         if (pre) {
             balances.push({
-                transactionIndex,
+                transactionIndex: ctx.transactionIndex,
                 account,
 
                 preProgramId: pre.programId ?? undefined,
@@ -501,7 +218,7 @@ function mapTokenBalances(
             })
         } else {
             balances.push({
-                transactionIndex,
+                transactionIndex: ctx.transactionIndex,
                 account,
                 postProgramId: post.programId ?? undefined,
                 postMint: post.mint,
@@ -515,7 +232,7 @@ function mapTokenBalances(
     for (let [account, pre] of preBalances.entries()) {
         if (postBalances.has(account)) continue
         balances.push({
-            transactionIndex,
+            transactionIndex: ctx.transactionIndex,
             account,
             preProgramId: pre.programId ?? undefined,
             preMint: pre.mint,