@subsquid/solana-stream 0.2.0-2-0.a5e4b7 → 0.2.0

package/src/instruction.ts

@@ -1,23 +1,28 @@
+import {Base58Bytes} from '@subsquid/solana-rpc-data'
 import {toHex} from '@subsquid/util-internal-hex'
-import {Bytes} from '@subsquid/util-internal'
 import bs58 from 'bs58'
+import {Bytes} from './data/model'
+
 
 export const DATA_SYM = Symbol('DATA')
 export const D8_SYM = Symbol('D8')
 
+
 interface Instruction {
-    data: Bytes
+    data: Base58Bytes
     [DATA_SYM]?: Uint8Array
     [D8_SYM]?: Bytes
 }
 
+
 export function getInstructionData(i: Instruction): Uint8Array {
     if (i[DATA_SYM]) return i[DATA_SYM]
-    return (i[DATA_SYM] = bs58.decode(i.data))
+    return i[DATA_SYM] = bs58.decode(i.data)
 }
 
+
 export function getInstructionDescriptor(i: Instruction): string {
     if (i[D8_SYM]) return i[D8_SYM]
     let bytes = toHex(getInstructionData(i))
-    return (i[D8_SYM] = bytes.slice(0, 18))
+    return i[D8_SYM] = bytes.slice(0, 18)
 }

package/src/rpc/client.ts

@@ -0,0 +1,26 @@
+import {RpcClient, RpcClientOptions} from '@subsquid/rpc-client'
+
+
+export interface SolanaRpcClientOptions extends RpcClientOptions {
+    /**
+     * Retry on connection error.
+     *
+     * By default, retries indefinitely.
+     */
+    retryAttempts?: number
+    /**
+     * Unsafe integer check must be always enabled
+     */
+    fixUnsafeIntegers?: true
+}
+
+
+export class SolanaRpcClient extends RpcClient {
+    constructor(options: SolanaRpcClientOptions) {
+        super({
+            retryAttempts: Number.MAX_SAFE_INTEGER,
+            ...options,
+            fixUnsafeIntegers: true
+        })
+    }
+}

package/src/rpc/filter.ts

@@ -0,0 +1,368 @@
+import {
+    Balance,
+    Block,
+    Instruction,
+    LogMessage,
+    Reward,
+    TokenBalance,
+    Transaction
+} from '@subsquid/solana-normalization'
+import {bisect, def, groupBy, weakMemo} from '@subsquid/util-internal'
+import {EntityFilter, FilterBuilder} from '@subsquid/util-internal-processor-tools'
+import assert from 'assert'
+import {
+    BalanceRequestRelations,
+    DataRequest,
+    InstructionRequestRelations,
+    LogRequestRelations,
+    TokenBalanceRequestRelations,
+    TransactionRequestRelations
+} from '../data/request'
+import {getInstructionDescriptor} from '../instruction'
+
+
+function buildTransactionFilter(dataRequest: DataRequest): EntityFilter<Transaction, TransactionRequestRelations> {
+    let items = new EntityFilter()
+    for (let req of dataRequest.transactions ?? []) {
+        let filter = new FilterBuilder<Transaction>()
+        let where = req.where || {}
+        filter.getIn(tx => tx.accountKeys[0], where.feePayer)
+        items.add(filter, req.include || {})
+    }
+    return items
+}
+
+
+function buildInstructionFilter(dataRequest: DataRequest): EntityFilter<Instruction, InstructionRequestRelations> {
+    let items = new EntityFilter()
+    for (let req of dataRequest.instructions ?? []) {
+        let filter = new FilterBuilder<Instruction>()
+        let where = req.where || {}
+        filter.propIn('programId', where.programId)
+        filter.matchAny((i, d) => getInstructionDescriptor(i).startsWith(d), where.d1)
+        filter.matchAny((i, d) => getInstructionDescriptor(i).startsWith(d), where.d2)
+        filter.matchAny((i, d) => getInstructionDescriptor(i).startsWith(d), where.d4)
+        filter.getIn(getInstructionDescriptor, where.d8)
+        filter.getIn(i => i.accounts[0], where.a0)
+        filter.getIn(i => i.accounts[1], where.a1)
+        filter.getIn(i => i.accounts[2], where.a2)
+        filter.getIn(i => i.accounts[3], where.a3)
+        filter.getIn(i => i.accounts[4], where.a4)
+        filter.getIn(i => i.accounts[5], where.a5)
+        filter.getIn(i => i.accounts[6], where.a6)
+        filter.getIn(i => i.accounts[7], where.a7)
+        filter.getIn(i => i.accounts[8], where.a8)
+        filter.getIn(i => i.accounts[9], where.a9)
+        if (where.isCommitted != null) {
+            filter.propIn('isCommitted', [where.isCommitted])
+        }
+        items.add(filter, req.include ?? {})
+    }
+    return items
+}
+
+
+function buildLogFilter(dataRequest: DataRequest): EntityFilter<LogMessage, LogRequestRelations> {
+    let items = new EntityFilter()
+    for (let req of dataRequest.logs ?? []) {
+        let filter = new FilterBuilder<LogMessage>()
+        let where = req.where ?? {}
+        filter.propIn('programId', where.programId)
+        filter.propIn('kind', where.kind)
+        items.add(filter, req.include ?? {})
+    }
+    return items
+}
+
+
+function buildBalanceFilter(dataRequest: DataRequest): EntityFilter<Balance, BalanceRequestRelations> {
+    let items = new EntityFilter()
+    for (let req of dataRequest.balances ?? []) {
+        let filter = new FilterBuilder<Balance>()
+        let where = req.where || {}
+        filter.propIn('account', where.account)
+        items.add(filter, req.include || {})
+    }
+    return items
+}
+
+
+function buildTokenBalanceFilter(dataRequest: DataRequest): EntityFilter<TokenBalance, TokenBalanceRequestRelations> {
+    let items = new EntityFilter()
+    for (let req of dataRequest.tokenBalances ?? []) {
+        let filter = new FilterBuilder<TokenBalance>()
+        let where = req.where || {}
+        filter.propIn('account', where.account)
+        filter.propIn('preProgramId', where.preProgramId)
+        filter.propIn('postProgramId', where.postProgramId)
+        filter.propIn('preMint', where.preMint)
+        filter.propIn('postMint', where.postMint)
+        filter.propIn('preOwner', where.preOwner)
+        filter.propIn('postOwner', where.postOwner)
+        items.add(filter, req.include || {})
+    }
+    return items
+}
+
+
+function buildRewardsFilter(dataRequest: DataRequest): EntityFilter<Reward, {}> {
+    let items = new EntityFilter()
+    for (let req of dataRequest.rewards ?? []) {
+        let filter = new FilterBuilder<Reward>()
+        let where = req.where || {}
+        filter.propIn('pubkey', where.pubkey)
+        items.add(filter, {})
+    }
+    return items
+}
+
+
+const getItemFilters = weakMemo((req: DataRequest) => {
+    return {
+        transactions: buildTransactionFilter(req),
+        instructions: buildInstructionFilter(req),
+        logs: buildLogFilter(req),
+        balances: buildBalanceFilter(req),
+        tokenBalances: buildTokenBalanceFilter(req),
+        rewards: buildRewardsFilter(req)
+    }
+})
+
+
+class IncludeSet {
+    transactions = new Set<Transaction>
+    instructions = new Set<Instruction>
+    logs = new Set<LogMessage>
+    balances = new Set<Balance>
+    tokenBalances = new Set<TokenBalance>
+    rewards = new Set<Reward>
+}
+
+
+export function filterBlockItems(block: Block, req: DataRequest): void {
+    new BlockFilter(block, req).apply()
+}
+
+
+class BlockFilter {
+    private include = new IncludeSet()
+    private items: ReturnType<typeof getItemFilters>
+
+    constructor(private block: Block, req: DataRequest) {
+        this.items = getItemFilters(req)
+    }
+
+    @def
+    private transactions(): Map<number, Transaction> {
+       return new Map(this.block.transactions.map(tx => [tx.transactionIndex, tx]))
+    }
+
+    private getTransaction(idx: number): Transaction {
+        let tx = this.transactions().get(idx)
+        assert(tx)
+        return tx
+    }
+
+    @def
+    private logsByTx(): Map<number, LogMessage[]> {
+        return groupBy(this.block.logs, log => log.transactionIndex)
+    }
+
+    @def
+    private instructionsByTx(): Map<number, Instruction[]> {
+        return groupBy(this.block.instructions, i => i.transactionIndex)
+    }
+
+    @def
+    private balancesByTx(): Map<number, Balance[]> {
+        return groupBy(this.block.balances, b => b.transactionIndex)
+    }
+
+    @def
+    private tokenBalancesByTx(): Map<number, TokenBalance[]> {
+        return groupBy(this.block.tokenBalances, b => b.transactionIndex)
+    }
+
+    private filterTransactions(): void {
+        if (!this.items.transactions.present()) return
+        for (let tx of this.block.transactions) {
+            let rel = this.items.transactions.match(tx)
+            if (rel == null) continue
+            this.include.transactions.add(tx)
+            if (rel.logs) {
+                include(this.include.logs, this.logsByTx().get(tx.transactionIndex))
+            }
+            if (rel.instructions) {
+                include(this.include.instructions, this.instructionsByTx().get(tx.transactionIndex))
+            }
+            if (rel.balances) {
+                include(this.include.balances, this.balancesByTx().get(tx.transactionIndex))
+            }
+            if (rel.tokenBalances) {
+                include(this.include.tokenBalances, this.tokenBalancesByTx().get(tx.transactionIndex))
+            }
+        }
+    }
+
+    private filterInstructions(): void {
+        if (!this.items.instructions.present()) return
+        for (let i = 0; i < this.block.instructions.length; i++) {
+            let ins = this.block.instructions[i]
+            let rel = this.items.instructions.match(ins)
+            if (rel == null) continue
+            this.include.instructions.add(ins)
+            if (rel.innerInstructions) {
+                for (let j = i + 1; j < this.block.instructions.length; j++) {
+                    let child = this.block.instructions[j]
+                    if (
+                        ins.transactionIndex == child.transactionIndex &&
+                        isChildAddress(ins.instructionAddress, child.instructionAddress)
+                    ) {
+                        this.include.instructions.add(child)
+                    } else {
+                        break
+                    }
+                }
+            }
+            if (rel.logs) {
+                let logs = this.logsByTx().get(ins.transactionIndex) ?? []
+                include(this.include.logs, findInstructionChildren(logs, ins.instructionAddress))
+            }
+            if (rel.transaction) {
+                this.include.transactions.add(this.getTransaction(ins.transactionIndex))
+            }
+            if (rel.transactionBalances) {
+                include(this.include.balances, this.balancesByTx().get(ins.transactionIndex))
+            }
+            if (rel.transactionTokenBalances) {
+                include(this.include.tokenBalances, this.tokenBalancesByTx().get(ins.transactionIndex))
+            }
+            if (rel.transactionInstructions) {
+                include(this.include.instructions, this.instructionsByTx().get(ins.transactionIndex))
+            }
+        }
+    }
+
+    private filterLogs(): void {
+        if (!this.items.logs.present()) return
+        for (let log of this.block.logs) {
+            let rel = this.items.logs.match(log)
+            if (rel == null) continue
+            this.include.logs.add(log)
+            if (rel.transaction) {
+                this.include.transactions.add(this.getTransaction(log.transactionIndex))
+            }
+            if (rel.instruction) {
+                this.include.instructions.add(this.getInstruction(log.transactionIndex, log.instructionAddress))
+            }
+        }
+    }
+
+    private getInstruction(transactionIdx: number, address: number[]): Instruction {
+        let pos = bisect(
+            this.block.instructions,
+            null,
+            ins => ins.transactionIndex - transactionIdx || addressCompare(ins.instructionAddress, address)
+        )
+        let ins = this.block.instructions[pos]
+        assert(
+            ins &&
+            ins.transactionIndex == transactionIdx &&
+            addressCompare(ins.instructionAddress, address) == 0
+        )
+        return ins
+    }
+
+    private filterTokenBalances(): void {
+        if (!this.items.tokenBalances.present()) return
+        for (let balance of this.block.tokenBalances) {
+            let rel = this.items.tokenBalances.match(balance)
+            if (rel == null) continue
+            this.include.tokenBalances.add(balance)
+            if (rel.transaction) {
+                this.include.transactions.add(this.getTransaction(balance.transactionIndex))
+            }
+            if (rel.transactionInstructions) {
+                include(this.include.instructions, this.instructionsByTx().get(balance.transactionIndex))
+            }
+        }
+    }
+
+    private filterBalances(): void {
+        if (!this.items.balances.present()) return
+        for (let balance of this.block.balances) {
+            let rel = this.items.balances.match(balance)
+            if (rel == null) continue
+            this.include.balances.add(balance)
+            if (rel.transaction) {
+                this.include.transactions.add(this.getTransaction(balance.transactionIndex))
+            }
+            if (rel.transactionInstructions) {
+                include(this.include.instructions, this.instructionsByTx().get(balance.transactionIndex))
+            }
+        }
+    }
+
+    private filterRewards(): void {
+        if (!this.items.rewards.present() || this.block.rewards == null) return
+        for (let reward of this.block.rewards) {
+            if (this.items.rewards.match(reward)) {
+                this.include.rewards.add(reward)
+            }
+        }
+    }
+
+    apply(): void {
+        this.filterTransactions()
+        this.filterInstructions()
+        this.filterLogs()
+        this.filterBalances()
+        this.filterTokenBalances()
+        this.filterRewards()
+        this.block.transactions = this.block.transactions.filter(i => this.include.transactions.has(i))
+        this.block.instructions = this.block.instructions.filter(i => this.include.instructions.has(i))
+        this.block.logs = this.block.logs.filter(i => this.include.logs.has(i))
+        this.block.balances = this.block.balances.filter(i => this.include.balances.has(i))
+        this.block.tokenBalances = this.block.tokenBalances.filter(i => this.include.tokenBalances.has(i))
+        this.block.rewards = this.block.rewards.filter(i => this.include.rewards.has(i))
+    }
+}
+
+
+function include<T>(set: Set<T>, items?: Iterable<T>): void {
+    if (items == null) return
+    for (let it of items) {
+        set.add(it)
+    }
+}
+
+
+function* findInstructionChildren<I extends {instructionAddress: number[]}>(
+    items: I[],
+    addr: number[],
+): Iterable<I> {
+    if (items.length == 0) return
+    let beg = bisect(items, addr, (ins, addr) => addressCompare(ins.instructionAddress, addr))
+    while (beg < items.length && isChildAddress(items[beg].instructionAddress, addr)) {
+        yield items[beg]
+        beg += 1
+    }
+}
+
+
+function isChildAddress(parent: number[], addr: number[]): boolean {
+    if (parent.length > addr.length) return false
+    for (let i = 0; i < parent.length; i++) {
+        if (parent[i] !== addr[i]) return false
+    }
+    return true
+}
+
+
+function addressCompare(a: number[], b: number[]): number {
+    for (let i = 0; i < Math.min(a.length, b.length); i++) {
+        let order = a[i] - b[i]
+        if (order) return order
+    }
+    return a.length - b.length
+}

package/src/rpc/mapping.ts

@@ -0,0 +1,13 @@
+import {mapRpcBlock} from '@subsquid/solana-normalization'
+import type * as rpc from '@subsquid/solana-rpc-data'
+import {PartialBlock} from '../data/partial'
+import {DataRequest} from '../data/request'
+import {filterBlockItems} from './filter'
+import {projectFields} from './project'
+
+
+export function mapBlock(src: rpc.Block, req: DataRequest): PartialBlock {
+    let block = mapRpcBlock(src)
+    filterBlockItems(block, req)
+    return projectFields(block, req.fields || {})
+}

package/src/rpc/project.ts

@@ -0,0 +1,61 @@
+import {Block} from '@subsquid/solana-normalization'
+import {project} from '../data/fields'
+import {FieldSelection} from '../data/model'
+import {PartialBlock} from '../data/partial'
+import {D8_SYM, DATA_SYM} from '../instruction'
+
+
+export function projectFields(block: Block, fields: FieldSelection): PartialBlock {
+    return {
+        header: {
+            height: block.header.height,
+            hash: block.header.hash,
+            parentHash: block.header.parentHash,
+            ...project(fields.block, block.header)
+        },
+        transactions: block.transactions.map(tx => {
+            return {
+                transactionIndex: tx.transactionIndex,
+                ...project(fields.transaction, tx)
+            }
+        }),
+        instructions: block.instructions.map(i => {
+            let ins = {
+                transactionIndex: i.transactionIndex,
+                instructionAddress: i.instructionAddress,
+                ...project(fields.instruction, i)
+            }
+            ;(ins as any)[D8_SYM] = (i as any)[D8_SYM]
+            ;(ins as any)[DATA_SYM] = (i as any)[DATA_SYM]
+            return ins
+        }),
+        logs: block.logs.map(log => {
+            return {
+                transactionIndex: log.transactionIndex,
+                logIndex: log.logIndex,
+                instructionAddress: log.instructionAddress,
+                ...project(fields.log, log)
+            }
+        }),
+        balances: block.balances.map(b => {
+            return {
+                transactionIndex: b.transactionIndex,
+                account: b.account,
+                ...project(fields.balance, b)
+            }
+        }),
+        tokenBalances: block.tokenBalances.map(b => {
+            return {
+                transactionIndex: b.transactionIndex,
+                account: b.account,
+                ...project(fields.tokenBalance, b)
+            }
+        }),
+        rewards: block.rewards.map(r => {
+            return {
+                pubkey: r.pubkey,
+                ...project(fields.reward, r)
+            }
+        })
+    }
+}

package/src/rpc/source.ts

@@ -0,0 +1,90 @@
+import {Base58Bytes, BlockInfo, findSlot, GetBlock, ingestFinalizedBlocks, Rpc} from '@subsquid/solana-rpc'
+import {addErrorContext, wait} from '@subsquid/util-internal'
+import {getRequestAt, mapRangeRequestList, RangeRequestList} from '@subsquid/util-internal-range'
+import {PartialBlock} from '../data/partial'
+import {DataRequest} from '../data/request'
+import type {RpcSettings} from '../source'
+import {mapBlock} from './mapping'
+
+
+export class RpcDataSource {
+    private rpc: Rpc
+
+    constructor(private options: RpcSettings) {
+        this.rpc = new Rpc(options.client)
+    }
+
+    getFinalizedHeight(): Promise<number> {
+        return this.rpc.getFinalizedHeight()
+    }
+
+    async getBlockInfo(slot: number): Promise<BlockInfo | undefined> {
+        let attempts = 10
+        while (attempts) {
+            let block = await this.rpc.getBlockInfo('finalized', slot)
+            if (block || block === undefined) return block
+            await wait(100)
+            attempts -= 1
+        }
+        throw new Error(`Failed to getBlock with finalized commitment at slot ${slot} 10 times in a row`)
+    }
+
+    async getBlockHash(height: number): Promise<Base58Bytes | undefined> {
+        let headSlot = await this.rpc.getTopSlot('finalized')
+        let top = {
+            slot: headSlot,
+            height: await this.rpc.getFinalizedBlockHeight(headSlot)
+        }
+        if (top.height < height) return
+        let bottom = {slot: 0, height: 0}
+        let slot = await findSlot(this.rpc, height, bottom, top)
+        let block = await this.rpc.getFinalizedBlockInfo(slot)
+        return block.blockhash
+    }
+
+    async *getBlockStream(
+        requests: RangeRequestList<DataRequest>,
+        stopOnHead?: boolean | undefined
+    ): AsyncIterable<PartialBlock[]> {
+        let blockStream = ingestFinalizedBlocks({
+            requests: toRpcRequests(requests),
+            stopOnHead,
+            rpc: this.rpc,
+            headPollInterval: 5_000,
+            strideSize: this.options.strideSize ?? 5,
+            strideConcurrency: this.options.strideConcurrency ?? 5,
+            concurrentFetchThreshold: this.options.concurrentFetchThreshold ?? 50
+        })
+
+        for await (let batch of blockStream) {
+            let req = getRequestAt(requests, batch[0].height) || {}
+            yield batch.map(block => {
+                try {
+                    return mapBlock(block, req)
+                } catch(err: any) {
+                    throw addErrorContext(err, {
+                        blockHash: block.hash,
+                        blockHeight: block.height,
+                        blockSlot: block.slot
+                    })
+                }
+            })
+        }
+    }
+}
+
+
+function toRpcRequests(requests: RangeRequestList<DataRequest>) {
+    return mapRangeRequestList(requests, req => {
+        let rewards = req.rewards?.length
+        let transactions = req.instructions?.length
+            || req.transactions?.length
+            || req.tokenBalances?.length
+            || req.balances?.length
+            || req.logs?.length
+        return {
+            rewards: !!rewards,
+            transactions: !!transactions
+        }
+    })
+}