@subsquid/solana-stream 0.0.0

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@subsquid/solana-stream",
+  "version": "0.0.0",
+  "description": "Solana block data streamer",
+  "license": "GPL-3.0-or-later",
+  "repository": "git@github.com:subsquid/squid.git",
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "lib",
+    "src"
+  ],
+  "main": "lib/index.js",
+  "dependencies": {
+    "@subsquid/http-client": "^1.4.0",
+    "@subsquid/logger": "^1.3.3",
+    "@subsquid/rpc-client": "^4.9.0",
+    "@subsquid/solana-normalization": "^0.0.0",
+    "@subsquid/solana-rpc": "^0.0.0",
+    "@subsquid/solana-rpc-data": "^0.0.0",
+    "@subsquid/util-internal": "^3.2.0",
+    "@subsquid/util-internal-archive-client": "^0.1.2",
+    "@subsquid/util-internal-hex": "^1.2.2",
+    "@subsquid/util-internal-ingest-tools": "^1.1.2",
+    "@subsquid/util-internal-processor-tools": "^4.1.1",
+    "@subsquid/util-internal-range": "^0.3.0",
+    "@subsquid/util-internal-validation": "^0.4.0",
+    "bs58": "^5.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^18.18.14",
+    "typescript": "~5.3.2"
+  },
+  "scripts": {
+    "build": "rm -rf lib && tsc"
+  }
+}

package/src/archive/schema.ts

@@ -0,0 +1,137 @@
+import {AddressTableLookup} from '@subsquid/solana-rpc-data'
+import {weakMemo} from '@subsquid/util-internal'
+import {
+    ANY_OBJECT,
+    array,
+    B58,
+    BIG_NAT,
+    BOOLEAN,
+    BYTES,
+    constant,
+    NAT,
+    nullable,
+    object,
+    oneOf,
+    option,
+    STRING
+} from '@subsquid/util-internal-validation'
+import {project} from '../data/fields'
+import {FieldSelection} from '../data/model'
+
+
+export const getDataSchema = weakMemo((fields: FieldSelection) => {
+    let BlockHeader = object({
+        number: NAT,
+        hash: B58,
+        parentHash: B58,
+        ...project(fields.block, {
+            slot: NAT,
+            parentSlot: NAT,
+            timestamp: NAT
+        })
+    })
+
+    let Transaction = object({
+        transactionIndex: NAT,
+        ...project(fields.transaction, {
+            version: oneOf({
+                legacy: constant('legacy'),
+                versionNumber: NAT
+            }),
+            accountKeys: array(B58),
+            addressTableLookups: array(AddressTableLookup),
+            numReadonlySignedAccounts: NAT,
+            numReadonlyUnsignedAccounts: NAT,
+            numRequiredSignatures: NAT,
+            recentBlockhash: B58,
+            signatures: array(B58),
+            err: nullable(ANY_OBJECT),
+            computeUnitsConsumed: BIG_NAT,
+            fee: BIG_NAT,
+            loadedAddresses: option(object({
+                readonly: array(B58),
+                writable: array(B58)
+            })),
+            hasDroppedLogMessages: BOOLEAN
+        })
+    })
+
+    let Instruction = object({
+        transactionIndex: NAT,
+        instructionAddress: array(NAT),
+        ...project(fields.instruction, {
+            programId: B58,
+            accounts: array(B58),
+            data: B58,
+            computeUnitsConsumed: option(BIG_NAT),
+            d1: BYTES,
+            d2: BYTES,
+            d4: BYTES,
+            d8: BYTES,
+            error: option(STRING),
+            isCommitted: BOOLEAN,
+            hasDroppedLogMessages: BOOLEAN
+        })
+    })
+
+    let LogMessage = object({
+        transactionIndex: NAT,
+        logIndex: NAT,
+        instructionAddress: array(NAT),
+        ...project(fields.log, {
+            programId: B58,
+            kind: oneOf({
+                log: constant('log'),
+                data: constant('data'),
+                other: constant('other')
+            }),
+            message: STRING
+        })
+    })
+
+    let Balance = object({
+        transactionIndex: NAT,
+        account: B58,
+        ...project(fields.balance, {
+            pre: BIG_NAT,
+            post: BIG_NAT
+        })
+    })
+
+    let TokenBalance = object({
+        transactionIndex: NAT,
+        account: B58,
+        ...project(fields.tokenBalance, {
+            preProgramId: option(B58),
+            postProgramId: option(B58),
+            preMint: option(B58),
+            postMint: option(B58),
+            preDecimals: option(NAT),
+            postDecimals: option(NAT),
+            preOwner: option(B58),
+            postOwner: option(B58),
+            preAmount: option(BIG_NAT),
+            postAmount: option(BIG_NAT)
+        })
+    })
+
+    let Reward = object({
+        pubkey: B58,
+        ...project(fields.reward, {
+            lamports: BIG_NAT,
+            postBalance: BIG_NAT,
+            rewardType: option(STRING),
+            commission: option(NAT)
+        })
+    })
+
+    return object({
+        header: BlockHeader,
+        transactions: option(array(Transaction)),
+        instructions: option(array(Instruction)),
+        logs: option(array(LogMessage)),
+        balances: option(array(Balance)),
+        tokenBalances: option(array(TokenBalance)),
+        rewards: option(array(Reward))
+    })
+})

package/src/archive/source.ts

@@ -0,0 +1,105 @@
+import {BlockHeader} from '@subsquid/solana-normalization'
+import {Base58Bytes} from '@subsquid/solana-rpc-data'
+import {assertNotNull} from '@subsquid/util-internal'
+import {ArchiveClient} from '@subsquid/util-internal-archive-client'
+import {archiveIngest} from '@subsquid/util-internal-ingest-tools'
+import {getRequestAt, mapRangeRequestList, RangeRequestList} from '@subsquid/util-internal-range'
+import {array, cast} from '@subsquid/util-internal-validation'
+import assert from 'assert'
+import {PartialBlock} from '../data/partial'
+import {DataRequest} from '../data/request'
+import {getDataSchema} from './schema'
+
+
+export class SolanaArchive {
+    constructor(private client: ArchiveClient) {}
+
+    getFinalizedHeight(): Promise<number> {
+        return this.client.getHeight()
+    }
+
+    async getBlockHash(height: number): Promise<Base58Bytes | undefined> {
+        let blocks = await this.client.query({
+            type: 'solana',
+            fromBlock: height,
+            toBlock: height,
+            includeAllBlocks: true
+        })
+        assert(blocks.length == 1)
+        return blocks[0].header.hash
+    }
+
+    async getBlockHeader(height: number): Promise<BlockHeader> {
+        let blocks = await this.client.query({
+            type: 'solana',
+            fromBlock: height,
+            toBlock: height,
+            includeAllBlocks: true,
+            fields: {
+                block: {
+                    slot: true,
+                    parentSlot: true,
+                    parentHash: true,
+                    timestamp: true
+                }
+            }
+        })
+        assert(blocks.length == 1)
+        let {number, ...rest} = blocks[0].header
+        return {
+            height: number,
+            ...rest
+        } as BlockHeader
+    }
+
+    async *getBlockStream(requests: RangeRequestList<DataRequest>, stopOnHead?: boolean | undefined): AsyncIterable<PartialBlock[]> {
+        let archiveRequests = mapRangeRequestList(requests, req => {
+            let {fields, includeAllBlocks, ...items} = req
+            let archiveItems: any = {}
+            let key: keyof typeof items
+            for (key in items) {
+                archiveItems[key] = items[key]?.map(it => ({...it.where, ...it.include}))
+            }
+            return {
+                type: 'solana',
+                fields: {
+                    block: {parentHash: true, ...fields?.block},
+                    transaction: fields?.transaction,
+                    instruction: fields?.instruction,
+                    log: {instructionAddress: true, ...fields?.log},
+                    balance: fields?.balance,
+                    tokenBalance: fields?.tokenBalance,
+                    reward: fields?.reward
+                },
+                includeAllBlocks,
+                ...archiveItems
+            }
+        })
+
+        for await (let batch of archiveIngest({
+            client: this.client,
+            requests: archiveRequests,
+            stopOnHead
+        })) {
+            let req = getRequestAt(requests, batch.blocks[0].header.number)
+
+            let blocks = cast(
+                array(getDataSchema(assertNotNull(req?.fields))),
+                batch.blocks
+            ).map(b => {
+                let {header: {number, ...hdr}, ...items} = b
+                return {
+                    header: {height: number, ...hdr},
+                    transactions: items.transactions || [],
+                    instructions: items.instructions || [],
+                    logs: items.logs || [],
+                    balances: items.balances || [],
+                    tokenBalances: items.tokenBalances || [],
+                    rewards: items.rewards || []
+                }
+            })
+
+            yield blocks
+        }
+    }
+}

package/src/data/fields.ts

@@ -0,0 +1,50 @@
+import {DEFAULT_FIELDS, FieldSelection} from './model'
+import {Selector} from './type-util'
+
+
+/**
+ * Get effective set of selected fields.
+ */
+export function getFields(fields: FieldSelection | undefined): FieldSelection {
+    return {
+        block: merge(DEFAULT_FIELDS.block, fields?.block),
+        transaction: merge(DEFAULT_FIELDS.transaction, fields?.transaction),
+        instruction: merge(DEFAULT_FIELDS.instruction, fields?.instruction),
+        log: merge(DEFAULT_FIELDS.log, fields?.log),
+        balance: merge(DEFAULT_FIELDS.balance, fields?.balance),
+        tokenBalance: merge(DEFAULT_FIELDS.tokenBalance, fields?.tokenBalance),
+        reward: merge(DEFAULT_FIELDS.reward, fields?.reward)
+    }
+}
+
+
+function merge<Keys extends string>(def: Selector<Keys>, requested: Selector<Keys> = {}): Selector<Keys> {
+    let fields: Selector<Keys> = {}
+
+    for (let key in def) {
+        if (requested[key] !== false) {
+            fields[key] = def[key]
+        }
+    }
+
+    for (let key in requested) {
+        if (requested[key]) {
+            fields[key] = true
+        }
+    }
+
+    return fields
+}
+
+
+export function project<T>(fields: Selector<keyof T> | undefined, obj: T): Partial<T> {
+    if (fields == null) return {}
+    let result: Partial<T> = {}
+    let key: keyof T
+    for (key in obj) {
+        if (fields[key]) {
+            result[key] = obj[key]
+        }
+    }
+    return result
+}

package/src/data/model.ts

@@ -0,0 +1,154 @@
+import type * as data from '@subsquid/solana-normalization'
+import type {
+    BalanceRequiredFields,
+    BlockRequiredFields,
+    InstructionRequiredFields,
+    LogRequiredFields,
+    RewardRequiredFields,
+    TokenBalanceRequiredFields,
+    TransactionRequiredFields
+} from './partial'
+import type {GetFields, Select, Selector, Simplify} from './type-util'
+
+
+/**
+ * Hex encoded binary string
+ */
+export type Bytes = string
+
+
+/**
+ * Base58 encoded binary string
+ */
+export type Base58Bytes = string
+
+
+export interface FieldSelection {
+    block?: Selector<Exclude<keyof data.BlockHeader, BlockRequiredFields>>
+    transaction?: Selector<Exclude<keyof data.Transaction, TransactionRequiredFields>>
+    instruction?: Selector<Exclude<keyof data.Instruction, InstructionRequiredFields>>
+    log?: Selector<Exclude<keyof data.LogMessage, LogRequiredFields>>
+    balance?: Selector<Exclude<keyof data.Balance, BalanceRequiredFields>>
+    tokenBalance?: Selector<Exclude<keyof data.TokenBalance, TokenBalanceRequiredFields>>
+    reward?: Selector<Exclude<keyof data.Reward, RewardRequiredFields>>
+}
+
+
+export const DEFAULT_FIELDS = {
+    block: {
+        slot: true,
+        parentSlot: true,
+        timestamp: true
+    },
+    transaction: {
+        signatures: true,
+        err: true
+    },
+    instruction: {
+        programId: true,
+        accounts: true,
+        data: true,
+        isCommitted: true
+    },
+    log: {
+        programId: true,
+        kind: true,
+        message: true
+    },
+    balance: {
+        pre: true,
+        post: true
+    },
+    tokenBalance: {
+        preMint: true,
+        preDecimals: true,
+        preOwner: true,
+        preAmount: true,
+        postMint: true,
+        postDecimals: true,
+        postOwner: true,
+        postAmount: true
+    },
+    reward: {
+        lamports: true,
+        rewardType: true
+    }
+} as const
+
+
+type Item<
+    Data,
+    RequiredFields extends keyof Data,
+    F extends FieldSelection,
+    K extends keyof FieldSelection
+> = Simplify<
+    Pick<Data, RequiredFields> &
+    Select<Data, GetFields<FieldSelection, typeof DEFAULT_FIELDS, F, K>>
+>
+
+
+export type BlockHeader<F extends FieldSelection = {}> = Item<
+    data.BlockHeader,
+    BlockRequiredFields,
+    F,
+    'block'
+>
+
+
+export type Transaction<F extends FieldSelection = {}> = Item<
+    data.Transaction,
+    TransactionRequiredFields,
+    F,
+    'transaction'
+>
+
+
+export type Instruction<F extends FieldSelection = {}> = Item<
+    data.Instruction,
+    InstructionRequiredFields,
+    F,
+    'instruction'
+>
+
+
+export type LogMessage<F extends FieldSelection = {}> = Item<
+    data.LogMessage,
+    LogRequiredFields,
+    F,
+    'log'
+>
+
+
+export type Balance<F extends FieldSelection = {}> = Item<
+    data.Balance,
+    BalanceRequiredFields,
+    F,
+    'balance'
+>
+
+
+export type TokenBalance<F extends FieldSelection = {}> = Item<
+    data.TokenBalance,
+    TokenBalanceRequiredFields,
+    F,
+    'tokenBalance'
+>
+
+
+export type Reward<F extends FieldSelection = {}> = Item<
+    data.Reward,
+    RewardRequiredFields,
+    F,
+    'reward'
+>
+
+
+export interface Block<F extends FieldSelection = {}> {
+    header: BlockHeader<F>
+    transactions: Transaction<F>[]
+    instructions: Instruction<F>[]
+    logs: LogMessage<F>[]
+    balances: Balance<F>[]
+    tokenBalances: TokenBalance<F>[]
+    rewards: Reward<F>[]
+}

package/src/data/partial.ts

@@ -0,0 +1,31 @@
+import type * as data from '@subsquid/solana-normalization'
+import type {MakePartial} from './type-util'
+
+
+export type BlockRequiredFields = 'height' | 'hash' | 'parentHash'
+export type TransactionRequiredFields = 'transactionIndex'
+export type InstructionRequiredFields = 'transactionIndex' | 'instructionAddress'
+export type LogRequiredFields = 'transactionIndex' | 'logIndex' | 'instructionAddress'
+export type BalanceRequiredFields = 'transactionIndex' | 'account'
+export type TokenBalanceRequiredFields = 'transactionIndex' | 'account'
+export type RewardRequiredFields = 'pubkey'
+
+
+export type PartialBlockHeader = MakePartial<data.BlockHeader, BlockRequiredFields>
+export type PartialTransaction = MakePartial<data.Transaction, TransactionRequiredFields>
+export type PartialInstruction = MakePartial<data.Instruction, InstructionRequiredFields>
+export type PartialLogMessage = MakePartial<data.LogMessage, LogRequiredFields>
+export type PartialBalance = MakePartial<data.Balance, BalanceRequiredFields>
+export type PartialTokenBalance = MakePartial<data.PrePostTokenBalance, TokenBalanceRequiredFields>
+export type PartialReward = MakePartial<data.Reward, RewardRequiredFields>
+
+
+export interface PartialBlock {
+    header: PartialBlockHeader
+    transactions: PartialTransaction[]
+    instructions: PartialInstruction[]
+    logs: PartialLogMessage[]
+    balances: PartialBalance[]
+    tokenBalances: PartialTokenBalance[]
+    rewards: PartialReward[]
+}

package/src/data/request.ts

@@ -0,0 +1,138 @@
+import {Base58Bytes} from '@subsquid/solana-rpc'
+import {LogMessage} from '@subsquid/solana-normalization'
+import {FieldSelection} from './model'
+
+
+export interface DataRequest {
+    fields?: FieldSelection
+    includeAllBlocks?: boolean
+    transactions?: TransactionRequest[]
+    instructions?: InstructionRequest[]
+    logs?: LogRequest[]
+    balances?: BalanceRequest[]
+    tokenBalances?: TokenBalanceRequest[]
+    rewards?: RewardRequest[]
+}
+
+
+export interface TransactionRequest {
+    where?: TransactionRequestWhere
+    include?: TransactionRequestRelations
+}
+
+
+export interface TransactionRequestWhere {
+    feePayer?: Base58Bytes[]
+}
+
+
+export interface TransactionRequestRelations {
+    instructions?: boolean
+    logs?: boolean
+}
+
+/**
+ * Hex encoded prefix of instruction data
+ */
+export type Discriminator = string
+
+
+export interface InstructionRequest {
+    where?: InstructionRequestWhere
+    include?: InstructionRequestRelations
+}
+
+
+export interface InstructionRequestWhere {
+    programId?: Base58Bytes[]
+    d1?: Discriminator[]
+    d2?: Discriminator[]
+    d3?: Discriminator[]
+    d4?: Discriminator[]
+    d8?: Discriminator[]
+    a0?: Base58Bytes[]
+    a1?: Base58Bytes[]
+    a2?: Base58Bytes[]
+    a3?: Base58Bytes[]
+    a4?: Base58Bytes[]
+    a5?: Base58Bytes[]
+    a6?: Base58Bytes[]
+    a7?: Base58Bytes[]
+    a8?: Base58Bytes[]
+    a9?: Base58Bytes[]
+    isCommitted?: boolean
+}
+
+
+export interface InstructionRequestRelations {
+    transaction?: boolean
+    transactionTokenBalances?: boolean
+    logs?: boolean
+    innerInstructions?: boolean
+}
+
+
+export interface LogRequest {
+    where?: LogRequestWhere
+    include?: LogRequestRelations
+}
+
+
+export interface LogRequestWhere {
+    programId?: Base58Bytes[]
+    kind?: LogMessage['kind'][]
+}
+
+
+export interface LogRequestRelations {
+    transaction?: boolean
+    instruction?: boolean
+}
+
+
+export interface BalanceRequest {
+    where?: BalanceRequestWhere
+    include?: BalanceRequestRelations
+}
+
+
+export interface BalanceRequestWhere {
+    account?: Base58Bytes[]
+}
+
+
+export interface BalanceRequestRelations {
+    transaction?: boolean
+    transactionInstructions?: boolean
+}
+
+
+export interface TokenBalanceRequest {
+    where?: TokenBalanceRequestWhere
+    include?: TokenBalanceRequestRelations
+}
+
+
+export interface TokenBalanceRequestWhere {
+    account?: Base58Bytes[]
+    preProgramId?: Base58Bytes[]
+    postProgramId?: Base58Bytes[]
+    preMint?: Base58Bytes[]
+    postMint?: Base58Bytes[]
+    preOwner?: Base58Bytes[]
+    postOwner?: Base58Bytes[]
+}
+
+
+export interface TokenBalanceRequestRelations {
+    transaction?: boolean
+    transactionInstructions?: boolean
+}
+
+
+export interface RewardRequest {
+    where?: {
+        pubkey?: Base58Bytes[]
+    }
+    include?: undefined
+}

package/src/data/type-util.ts

@@ -0,0 +1,42 @@
+export type Simplify<T> = {
+    [K in keyof T]: T[K]
+} & {}
+
+
+export type ExcludeUndefined<T> = {
+    [K in keyof T as undefined extends T[K] ? never : K]: T[K]
+} & {}
+
+
+export type GetFields<
+    FieldSelectionType,
+    Defaults extends FieldSelectionType,
+    Selection extends FieldSelectionType,
+    K extends keyof FieldSelectionType
+> = TrueFields<MergeDefault<Selection[K], Defaults[K]>>
+
+
+type MergeDefault<T, D> = Simplify<
+    undefined extends T ? D : Omit<D, keyof ExcludeUndefined<T>> & ExcludeUndefined<T>
+>
+
+
+type TrueFields<F> = keyof {
+    [K in keyof F as true extends F[K] ? K : never]: true
+}
+
+
+export type Select<T, Fields> = T extends any ? Simplify<Pick<T, Extract<keyof T, Fields>>> : never
+
+
+export type Selector<Fields extends string | number | symbol> = {
+    [P in Fields]?: boolean
+}
+
+
+export type MakePartial<T, Required extends keyof T> = Simplify<
+    Pick<T, Required> &
+    {
+        [K in keyof T as K extends Required ? never : K]+?: T[K]
+    }
+>

package/src/index.ts

@@ -0,0 +1,4 @@
+export * from './data/model'
+export * from './rpc/client'
+export * from './instruction'
+export * from './source'