@subsquid/solana-stream 0.2.0-2-0.a5e4b7 → 0.2.0

package/src/{schema.ts → archive/schema.ts}

@@ -1,4 +1,4 @@
-import type {Selector} from '@subsquid/util-internal'
+import {AddressTableLookup} from '@subsquid/solana-rpc-data'
 import {weakMemo} from '@subsquid/util-internal'
 import {
     ANY_OBJECT,
@@ -13,28 +13,11 @@ import {
     object,
     oneOf,
     option,
-    STRING,
-    withDefault,
+    STRING
 } from '@subsquid/util-internal-validation'
-import {FieldSelection} from './objects'
+import {project} from '../data/fields'
+import {FieldSelection} from '../data/model'
 
-export function project<T>(fields: Selector<keyof T> | undefined, obj: T): Partial<T> {
-    if (fields == null) return {}
-    let result: Partial<T> = {}
-    let key: keyof T
-    for (key in obj) {
-        if (fields[key]) {
-            result[key] = obj[key]
-        }
-    }
-    return result
-}
-
-export const AddressTableLookup = object({
-    accountKey: B58,
-    readonlyIndexes: array(NAT),
-    writableIndexes: array(NAT),
-})
 
 export const getDataSchema = weakMemo((fields: FieldSelection) => {
     let BlockHeader = object({
@@ -42,18 +25,18 @@ export const getDataSchema = weakMemo((fields: FieldSelection) => {
         hash: B58,
         parentHash: B58,
         ...project(fields.block, {
-            height: NAT,
+            slot: NAT,
             parentSlot: NAT,
-            timestamp: NAT,
-        }),
+            timestamp: NAT
+        })
     })
 
     let Transaction = object({
+        transactionIndex: NAT,
         ...project(fields.transaction, {
-            transactionIndex: NAT,
             version: oneOf({
                 legacy: constant('legacy'),
-                versionNumber: NAT,
+                versionNumber: NAT
             }),
             accountKeys: array(B58),
             addressTableLookups: array(AddressTableLookup),
@@ -65,20 +48,18 @@ export const getDataSchema = weakMemo((fields: FieldSelection) => {
             err: nullable(ANY_OBJECT),
             computeUnitsConsumed: BIG_NAT,
             fee: BIG_NAT,
-            loadedAddresses: option(
-                object({
-                    readonly: array(B58),
-                    writable: array(B58),
-                })
-            ),
-            hasDroppedLogMessages: BOOLEAN,
-        }),
+            loadedAddresses: option(object({
+                readonly: array(B58),
+                writable: array(B58)
+            })),
+            hasDroppedLogMessages: BOOLEAN
+        })
     })
 
     let Instruction = object({
+        transactionIndex: NAT,
+        instructionAddress: array(NAT),
         ...project(fields.instruction, {
-            transactionIndex: NAT,
-            instructionAddress: array(NAT),
             programId: B58,
             accounts: array(B58),
             data: B58,
@@ -89,8 +70,8 @@ export const getDataSchema = weakMemo((fields: FieldSelection) => {
             d8: BYTES,
             error: option(STRING),
             isCommitted: BOOLEAN,
-            hasDroppedLogMessages: BOOLEAN,
-        }),
+            hasDroppedLogMessages: BOOLEAN
+        })
     })
 
     let LogMessage = object({
@@ -102,25 +83,25 @@ export const getDataSchema = weakMemo((fields: FieldSelection) => {
             kind: oneOf({
                 log: constant('log'),
                 data: constant('data'),
-                other: constant('other'),
+                other: constant('other')
             }),
-            message: STRING,
-        }),
+            message: STRING
+        })
     })
 
     let Balance = object({
+        transactionIndex: NAT,
+        account: B58,
         ...project(fields.balance, {
-            transactionIndex: NAT,
-            account: B58,
             pre: BIG_NAT,
-            post: BIG_NAT,
-        }),
+            post: BIG_NAT
+        })
     })
 
     let TokenBalance = object({
+        transactionIndex: NAT,
+        account: B58,
         ...project(fields.tokenBalance, {
-            transactionIndex: NAT,
-            account: B58,
             preProgramId: option(B58),
             postProgramId: option(B58),
             preMint: option(B58),
@@ -130,27 +111,27 @@ export const getDataSchema = weakMemo((fields: FieldSelection) => {
             preOwner: option(B58),
             postOwner: option(B58),
             preAmount: option(BIG_NAT),
-            postAmount: option(BIG_NAT),
-        }),
+            postAmount: option(BIG_NAT)
+        })
     })
 
     let Reward = object({
+        pubkey: B58,
         ...project(fields.reward, {
-            pubkey: B58,
             lamports: BIG_NAT,
             postBalance: BIG_NAT,
             rewardType: option(STRING),
-            commission: option(NAT),
-        }),
+            commission: option(NAT)
+        })
     })
 
     return object({
         header: BlockHeader,
-        transactions: withDefault([], array(Transaction)),
+        transactions: option(array(Transaction)),
         instructions: option(array(Instruction)),
         logs: option(array(LogMessage)),
         balances: option(array(Balance)),
         tokenBalances: option(array(TokenBalance)),
-        rewards: option(array(Reward)),
+        rewards: option(array(Reward))
     })
 })

package/src/archive/source.ts

@@ -0,0 +1,105 @@
+import {BlockHeader} from '@subsquid/solana-normalization'
+import {Base58Bytes} from '@subsquid/solana-rpc-data'
+import {assertNotNull} from '@subsquid/util-internal'
+import {ArchiveClient} from '@subsquid/util-internal-archive-client'
+import {archiveIngest} from '@subsquid/util-internal-ingest-tools'
+import {getRequestAt, mapRangeRequestList, RangeRequestList} from '@subsquid/util-internal-range'
+import {array, cast} from '@subsquid/util-internal-validation'
+import assert from 'assert'
+import {PartialBlock} from '../data/partial'
+import {DataRequest} from '../data/request'
+import {getDataSchema} from './schema'
+
+
+export class SolanaArchive {
+    constructor(private client: ArchiveClient) {}
+
+    getFinalizedHeight(): Promise<number> {
+        return this.client.getHeight()
+    }
+
+    async getBlockHash(height: number): Promise<Base58Bytes | undefined> {
+        let blocks = await this.client.query({
+            type: 'solana',
+            fromBlock: height,
+            toBlock: height,
+            includeAllBlocks: true
+        })
+        assert(blocks.length == 1)
+        return blocks[0].header.hash
+    }
+
+    async getBlockHeader(height: number): Promise<BlockHeader> {
+        let blocks = await this.client.query({
+            type: 'solana',
+            fromBlock: height,
+            toBlock: height,
+            includeAllBlocks: true,
+            fields: {
+                block: {
+                    slot: true,
+                    parentSlot: true,
+                    parentHash: true,
+                    timestamp: true
+                }
+            }
+        })
+        assert(blocks.length == 1)
+        let {number, ...rest} = blocks[0].header
+        return {
+            height: number,
+            ...rest
+        } as BlockHeader
+    }
+
+    async *getBlockStream(requests: RangeRequestList<DataRequest>, stopOnHead?: boolean | undefined): AsyncIterable<PartialBlock[]> {
+        let archiveRequests = mapRangeRequestList(requests, req => {
+            let {fields, includeAllBlocks, ...items} = req
+            let archiveItems: any = {}
+            let key: keyof typeof items
+            for (key in items) {
+                archiveItems[key] = items[key]?.map(it => ({...it.where, ...it.include}))
+            }
+            return {
+                type: 'solana',
+                fields: {
+                    block: {parentHash: true, ...fields?.block},
+                    transaction: fields?.transaction,
+                    instruction: fields?.instruction,
+                    log: {instructionAddress: true, ...fields?.log},
+                    balance: fields?.balance,
+                    tokenBalance: fields?.tokenBalance,
+                    reward: fields?.reward
+                },
+                includeAllBlocks,
+                ...archiveItems
+            }
+        })
+
+        for await (let batch of archiveIngest({
+            client: this.client,
+            requests: archiveRequests,
+            stopOnHead
+        })) {
+            let req = getRequestAt(requests, batch.blocks[0].header.number)
+
+            let blocks = cast(
+                array(getDataSchema(assertNotNull(req?.fields))),
+                batch.blocks
+            ).map(b => {
+                let {header: {number, ...hdr}, ...items} = b
+                return {
+                    header: {height: number, ...hdr},
+                    transactions: items.transactions || [],
+                    instructions: items.instructions || [],
+                    logs: items.logs || [],
+                    balances: items.balances || [],
+                    tokenBalances: items.tokenBalances || [],
+                    rewards: items.rewards || []
+                }
+            })
+
+            yield blocks
+        }
+    }
+}

package/src/data/fields.ts

@@ -0,0 +1,50 @@
+import {DEFAULT_FIELDS, FieldSelection} from './model'
+import {Selector} from './type-util'
+
+
+/**
+ * Get effective set of selected fields.
+ */
+export function getFields(fields: FieldSelection | undefined): FieldSelection {
+    return {
+        block: merge(DEFAULT_FIELDS.block, fields?.block),
+        transaction: merge(DEFAULT_FIELDS.transaction, fields?.transaction),
+        instruction: merge(DEFAULT_FIELDS.instruction, fields?.instruction),
+        log: merge(DEFAULT_FIELDS.log, fields?.log),
+        balance: merge(DEFAULT_FIELDS.balance, fields?.balance),
+        tokenBalance: merge(DEFAULT_FIELDS.tokenBalance, fields?.tokenBalance),
+        reward: merge(DEFAULT_FIELDS.reward, fields?.reward)
+    }
+}
+
+
+function merge<Keys extends string>(def: Selector<Keys>, requested: Selector<Keys> = {}): Selector<Keys> {
+    let fields: Selector<Keys> = {}
+
+    for (let key in def) {
+        if (requested[key] !== false) {
+            fields[key] = def[key]
+        }
+    }
+
+    for (let key in requested) {
+        if (requested[key]) {
+            fields[key] = true
+        }
+    }
+
+    return fields
+}
+
+
+export function project<T>(fields: Selector<keyof T> | undefined, obj: T): Partial<T> {
+    if (fields == null) return {}
+    let result: Partial<T> = {}
+    let key: keyof T
+    for (key in obj) {
+        if (fields[key]) {
+            result[key] = obj[key]
+        }
+    }
+    return result
+}

package/src/data/model.ts

@@ -0,0 +1,154 @@
+import type * as data from '@subsquid/solana-normalization'
+import type {
+    BalanceRequiredFields,
+    BlockRequiredFields,
+    InstructionRequiredFields,
+    LogRequiredFields,
+    RewardRequiredFields,
+    TokenBalanceRequiredFields,
+    TransactionRequiredFields
+} from './partial'
+import type {GetFields, Select, Selector, Simplify} from './type-util'
+
+
+/**
+ * Hex encoded binary string
+ */
+export type Bytes = string
+
+
+/**
+ * Base58 encoded binary string
+ */
+export type Base58Bytes = string
+
+
+export interface FieldSelection {
+    block?: Selector<Exclude<keyof data.BlockHeader, BlockRequiredFields>>
+    transaction?: Selector<Exclude<keyof data.Transaction, TransactionRequiredFields>>
+    instruction?: Selector<Exclude<keyof data.Instruction, InstructionRequiredFields>>
+    log?: Selector<Exclude<keyof data.LogMessage, LogRequiredFields>>
+    balance?: Selector<Exclude<keyof data.Balance, BalanceRequiredFields>>
+    tokenBalance?: Selector<Exclude<keyof data.TokenBalance, TokenBalanceRequiredFields>>
+    reward?: Selector<Exclude<keyof data.Reward, RewardRequiredFields>>
+}
+
+
+export const DEFAULT_FIELDS = {
+    block: {
+        slot: true,
+        parentSlot: true,
+        timestamp: true
+    },
+    transaction: {
+        signatures: true,
+        err: true
+    },
+    instruction: {
+        programId: true,
+        accounts: true,
+        data: true,
+        isCommitted: true
+    },
+    log: {
+        programId: true,
+        kind: true,
+        message: true
+    },
+    balance: {
+        pre: true,
+        post: true
+    },
+    tokenBalance: {
+        preMint: true,
+        preDecimals: true,
+        preOwner: true,
+        preAmount: true,
+        postMint: true,
+        postDecimals: true,
+        postOwner: true,
+        postAmount: true
+    },
+    reward: {
+        lamports: true,
+        rewardType: true
+    }
+} as const
+
+
+type Item<
+    Data,
+    RequiredFields extends keyof Data,
+    F extends FieldSelection,
+    K extends keyof FieldSelection
+> = Simplify<
+    Pick<Data, RequiredFields> &
+    Select<Data, GetFields<FieldSelection, typeof DEFAULT_FIELDS, F, K>>
+>
+
+
+export type BlockHeader<F extends FieldSelection = {}> = Item<
+    data.BlockHeader,
+    BlockRequiredFields,
+    F,
+    'block'
+>
+
+
+export type Transaction<F extends FieldSelection = {}> = Item<
+    data.Transaction,
+    TransactionRequiredFields,
+    F,
+    'transaction'
+>
+
+
+export type Instruction<F extends FieldSelection = {}> = Item<
+    data.Instruction,
+    InstructionRequiredFields,
+    F,
+    'instruction'
+>
+
+
+export type LogMessage<F extends FieldSelection = {}> = Item<
+    data.LogMessage,
+    LogRequiredFields,
+    F,
+    'log'
+>
+
+
+export type Balance<F extends FieldSelection = {}> = Item<
+    data.Balance,
+    BalanceRequiredFields,
+    F,
+    'balance'
+>
+
+
+export type TokenBalance<F extends FieldSelection = {}> = Item<
+    data.TokenBalance,
+    TokenBalanceRequiredFields,
+    F,
+    'tokenBalance'
+>
+
+
+export type Reward<F extends FieldSelection = {}> = Item<
+    data.Reward,
+    RewardRequiredFields,
+    F,
+    'reward'
+>
+
+
+export interface Block<F extends FieldSelection = {}> {
+    header: BlockHeader<F>
+    transactions: Transaction<F>[]
+    instructions: Instruction<F>[]
+    logs: LogMessage<F>[]
+    balances: Balance<F>[]
+    tokenBalances: TokenBalance<F>[]
+    rewards: Reward<F>[]
+}

package/src/data/partial.ts

@@ -0,0 +1,31 @@
+import type * as data from '@subsquid/solana-normalization'
+import type {MakePartial} from './type-util'
+
+
+export type BlockRequiredFields = 'height' | 'hash' | 'parentHash'
+export type TransactionRequiredFields = 'transactionIndex'
+export type InstructionRequiredFields = 'transactionIndex' | 'instructionAddress'
+export type LogRequiredFields = 'transactionIndex' | 'logIndex' | 'instructionAddress'
+export type BalanceRequiredFields = 'transactionIndex' | 'account'
+export type TokenBalanceRequiredFields = 'transactionIndex' | 'account'
+export type RewardRequiredFields = 'pubkey'
+
+
+export type PartialBlockHeader = MakePartial<data.BlockHeader, BlockRequiredFields>
+export type PartialTransaction = MakePartial<data.Transaction, TransactionRequiredFields>
+export type PartialInstruction = MakePartial<data.Instruction, InstructionRequiredFields>
+export type PartialLogMessage = MakePartial<data.LogMessage, LogRequiredFields>
+export type PartialBalance = MakePartial<data.Balance, BalanceRequiredFields>
+export type PartialTokenBalance = MakePartial<data.PrePostTokenBalance, TokenBalanceRequiredFields>
+export type PartialReward = MakePartial<data.Reward, RewardRequiredFields>
+
+
+export interface PartialBlock {
+    header: PartialBlockHeader
+    transactions: PartialTransaction[]
+    instructions: PartialInstruction[]
+    logs: PartialLogMessage[]
+    balances: PartialBalance[]
+    tokenBalances: PartialTokenBalance[]
+    rewards: PartialReward[]
+}

package/src/data/request.ts

@@ -0,0 +1,142 @@
+import {Base58Bytes} from '@subsquid/solana-rpc'
+import {LogMessage} from '@subsquid/solana-normalization'
+import {FieldSelection} from './model'
+
+
+export interface DataRequest {
+    fields?: FieldSelection
+    includeAllBlocks?: boolean
+    transactions?: TransactionRequest[]
+    instructions?: InstructionRequest[]
+    logs?: LogRequest[]
+    balances?: BalanceRequest[]
+    tokenBalances?: TokenBalanceRequest[]
+    rewards?: RewardRequest[]
+}
+
+
+export interface TransactionRequest {
+    where?: TransactionRequestWhere
+    include?: TransactionRequestRelations
+}
+
+
+export interface TransactionRequestWhere {
+    feePayer?: Base58Bytes[]
+}
+
+
+export interface TransactionRequestRelations {
+    instructions?: boolean
+    logs?: boolean
+    balances?: boolean
+    tokenBalances?: boolean
+}
+
+/**
+ * Hex encoded prefix of instruction data
+ */
+export type Discriminator = string
+
+
+export interface InstructionRequest {
+    where?: InstructionRequestWhere
+    include?: InstructionRequestRelations
+}
+
+
+export interface InstructionRequestWhere {
+    programId?: Base58Bytes[]
+    d1?: Discriminator[]
+    d2?: Discriminator[]
+    d3?: Discriminator[]
+    d4?: Discriminator[]
+    d8?: Discriminator[]
+    a0?: Base58Bytes[]
+    a1?: Base58Bytes[]
+    a2?: Base58Bytes[]
+    a3?: Base58Bytes[]
+    a4?: Base58Bytes[]
+    a5?: Base58Bytes[]
+    a6?: Base58Bytes[]
+    a7?: Base58Bytes[]
+    a8?: Base58Bytes[]
+    a9?: Base58Bytes[]
+    isCommitted?: boolean
+}
+
+
+export interface InstructionRequestRelations {
+    transaction?: boolean
+    transactionBalances?: boolean
+    transactionTokenBalances?: boolean
+    transactionInstructions?: boolean
+    logs?: boolean
+    innerInstructions?: boolean
+}
+
+
+export interface LogRequest {
+    where?: LogRequestWhere
+    include?: LogRequestRelations
+}
+
+
+export interface LogRequestWhere {
+    programId?: Base58Bytes[]
+    kind?: LogMessage['kind'][]
+}
+
+
+export interface LogRequestRelations {
+    transaction?: boolean
+    instruction?: boolean
+}
+
+
+export interface BalanceRequest {
+    where?: BalanceRequestWhere
+    include?: BalanceRequestRelations
+}
+
+
+export interface BalanceRequestWhere {
+    account?: Base58Bytes[]
+}
+
+
+export interface BalanceRequestRelations {
+    transaction?: boolean
+    transactionInstructions?: boolean
+}
+
+
+export interface TokenBalanceRequest {
+    where?: TokenBalanceRequestWhere
+    include?: TokenBalanceRequestRelations
+}
+
+
+export interface TokenBalanceRequestWhere {
+    account?: Base58Bytes[]
+    preProgramId?: Base58Bytes[]
+    postProgramId?: Base58Bytes[]
+    preMint?: Base58Bytes[]
+    postMint?: Base58Bytes[]
+    preOwner?: Base58Bytes[]
+    postOwner?: Base58Bytes[]
+}
+
+
+export interface TokenBalanceRequestRelations {
+    transaction?: boolean
+    transactionInstructions?: boolean
+}
+
+
+export interface RewardRequest {
+    where?: {
+        pubkey?: Base58Bytes[]
+    }
+    include?: undefined
+}

package/src/data/type-util.ts

@@ -0,0 +1,42 @@
+export type Simplify<T> = {
+    [K in keyof T]: T[K]
+} & {}
+
+
+export type ExcludeUndefined<T> = {
+    [K in keyof T as undefined extends T[K] ? never : K]: T[K]
+} & {}
+
+
+export type GetFields<
+    FieldSelectionType,
+    Defaults extends FieldSelectionType,
+    Selection extends FieldSelectionType,
+    K extends keyof FieldSelectionType
+> = TrueFields<MergeDefault<Selection[K], Defaults[K]>>
+
+
+type MergeDefault<T, D> = Simplify<
+    undefined extends T ? D : Omit<D, keyof ExcludeUndefined<T>> & ExcludeUndefined<T>
+>
+
+
+type TrueFields<F> = keyof {
+    [K in keyof F as true extends F[K] ? K : never]: true
+}
+
+
+export type Select<T, Fields> = T extends any ? Simplify<Pick<T, Extract<keyof T, Fields>>> : never
+
+
+export type Selector<Fields extends string | number | symbol> = {
+    [P in Fields]?: boolean
+}
+
+
+export type MakePartial<T, Required extends keyof T> = Simplify<
+    Pick<T, Required> &
+    {
+        [K in keyof T as K extends Required ? never : K]+?: T[K]
+    }
+>

package/src/index.ts

@@ -1,4 +1,4 @@
-export * from './source'
-export * from './query'
-export * from './objects'
+export * from './data/model'
+export * from './rpc/client'
 export * from './instruction'
+export * from './source'