@subsquid/fuel-stream 0.0.1

package/src/archive/data-schema.ts

@@ -0,0 +1,320 @@
+import {weakMemo} from '@subsquid/util-internal'
+import {
+    array,
+    BIG_NAT,
+    BOOLEAN,
+    BYTES,
+    NAT,
+    object,
+    option,
+    STRING,
+    taggedUnion,
+    oneOf,
+    constant
+} from '@subsquid/util-internal-validation'
+import {FieldSelection} from '../data/model'
+import {Selector} from '../data/util'
+
+
+export const getDataSchema = weakMemo((fields: FieldSelection) => {
+    let BlockHeader = object({
+        number: NAT,
+        hash: BYTES,
+        ...project(fields.block, {
+            daHeight: BIG_NAT,
+            transactionsRoot: BYTES,
+            transactionsCount: BIG_NAT,
+            messageReceiptRoot: BYTES,
+            messageReceiptCount: BIG_NAT,
+            applicationHash: BYTES,
+            prevRoot: BYTES,
+            time: BIG_NAT,
+        })
+    })
+
+    let Transaction = object({
+        index: NAT,
+        ...project(fields.transaction, {
+            bytecodeLength: option(BIG_NAT),
+            bytecodeWitnessIndex: option(NAT),
+            gasPrice: option(BIG_NAT),
+            hash: BYTES,
+            inputAssetIds: option(array(BYTES)),
+            policies: option(object({
+                gasPrice: option(BIG_NAT),
+                witnessLimit: option(BIG_NAT),
+                maturity: option(NAT),
+                maxFee: option(BIG_NAT)
+            })),
+            inputContract: option(object({
+                utxoId: BYTES,
+                balanceRoot: BYTES,
+                stateRoot: BYTES,
+                txPointer: STRING,
+                contract: BYTES
+            })),
+            inputContracts: option(array(BYTES)),
+            isCreate: BOOLEAN,
+            isMint: BOOLEAN,
+            isScript: BOOLEAN,
+            maturity: option(NAT),
+            mintAmount: option(BIG_NAT),
+            mintAssetId: option(BYTES),
+            outputContract: option(object({
+                inputIndex: NAT,
+                balanceRoot: BYTES,
+                stateRoot: BYTES
+            })),
+            rawPayload: option(BYTES),
+            receiptsRoot: option(BYTES),
+            salt: option(BYTES),
+            script: option(BYTES),
+            scriptData: option(BYTES),
+            scriptGasLimit: option(BIG_NAT),
+            storageSlots: option(array(BYTES)),
+            txPointer: option(STRING),
+            type: oneOf({
+                script: constant('Script'),
+                create: constant('Create'),
+                mint: constant('Mint'),
+            }),
+            witnesses: option(array(BYTES)),
+        })
+    })
+
+    let Receipt = object({
+        transactionIndex: NAT,
+        index: NAT,
+        ...project(fields.receipt, {
+            amount: option(BIG_NAT),
+            assetId: option(BYTES),
+            contract: option(BYTES),
+            contractId: option(BYTES),
+            data: option(BYTES),
+            digest: option(BYTES),
+            gas: option(BIG_NAT),
+            gasUsed: option(BIG_NAT),
+            is: option(BIG_NAT),
+            len: option(BIG_NAT),
+            nonce: option(BYTES),
+            param1: option(BIG_NAT),
+            param2: option(BIG_NAT),
+            pc: option(BIG_NAT),
+            ptr: option(BIG_NAT),
+            ra: option(BIG_NAT),
+            rb: option(BIG_NAT),
+            rc: option(BIG_NAT),
+            rd: option(BIG_NAT),
+            reason: option(BIG_NAT),
+            receiptType: oneOf({
+                CALL: constant('CALL'),
+                RETURN: constant('RETURN'),
+                RETURN_DATA: constant('RETURN_DATA'),
+                PANIC: constant('PANIC'),
+                REVERT: constant('REVERT'),
+                LOG: constant('LOG'),
+                LOG_DATA: constant('LOG_DATA'),
+                TRANSFER: constant('TRANSFER'),
+                TRANSFER_OUT: constant('TRANSFER_OUT'),
+                SCRIPT_RESULT: constant('SCRIPT_RESULT'),
+                MESSAGE_OUT: constant('MESSAGE_OUT'),
+                MINT: constant('MINT'),
+                BURN: constant('BURN'),
+            }),
+            recipient: option(BYTES),
+            result: option(BIG_NAT),
+            sender: option(BYTES),
+            subId: option(BYTES),
+            to: option(BYTES),
+            toAddress: option(BYTES),
+            val: option(BIG_NAT),
+        })
+    })
+
+    let InputCoin = object({
+        transactionIndex: NAT,
+        index: NAT,
+        ...project({
+            amount: fields.input?.coinAmount,
+            assetId: fields.input?.coinAssetId,
+            maturity: fields.input?.coinMaturity,
+            owner: fields.input?.coinOwner,
+            predicate: fields.input?.coinPredicate,
+            predicateData: fields.input?.coinPredicateData,
+            predicateGasUsed: fields.input?.coinPredicateGasUsed,
+            txPointer: fields.input?.coinTxPointer,
+            utxoId: fields.input?.coinUtxoId,
+            witnessIndex: fields.input?.coinWitnessIndex
+        }, {
+            utxoId: BYTES,
+            owner: BYTES,
+            amount: BIG_NAT,
+            assetId: BYTES,
+            txPointer: STRING,
+            witnessIndex: NAT,
+            maturity: NAT,
+            predicateGasUsed: BIG_NAT,
+            predicate: BYTES,
+            predicateData: BYTES,
+        })
+    })
+
+    let InputContract = object({
+        transactionIndex: NAT,
+        index: NAT,
+        ...project({
+            utxoId: fields.input?.contractUtxoId,
+            balanceRoot: fields.input?.contractBalanceRoot,
+            contract: fields.input?.contractContract,
+            stateRoot: fields.input?.contractStateRoot,
+            txPointer: fields.input?.contractTxPointer
+        }, {
+            utxoId: BYTES,
+            balanceRoot: BYTES,
+            stateRoot: BYTES,
+            txPointer: STRING,
+            contract: BYTES,
+        })
+    })
+
+    let InputMessage = object({
+        transactionIndex: NAT,
+        index: NAT,
+        ...project({
+            sender: fields.input?.messageSender,
+            amount: fields.input?.messageAmount,
+            data: fields.input?.messageData,
+            nonce: fields.input?.messageNonce,
+            predicate: fields.input?.messagePredicate,
+            predicateData: fields.input?.messagePredicateData,
+            predicateGasUsed: fields.input?.messagePredicateGasUsed,
+            recipient: fields.input?.messageRecipient,
+            witnessIndex: fields.input?.messageWitnessIndex
+        }, {
+            sender: BYTES,
+            recipient: BYTES,
+            amount: BIG_NAT,
+            nonce: BYTES,
+            witnessIndex: NAT,
+            predicateGasUsed: BIG_NAT,
+            data: BYTES,
+            predicate: BYTES,
+            predicateData: BYTES,
+        })
+    })
+
+    let Input = taggedUnion('type', {
+        InputCoin,
+        InputContract,
+        InputMessage
+    })
+
+    let CoinOutput = object({
+        transactionIndex: NAT,
+        index: NAT,
+        ...project({
+            amount: fields.output?.coinAmount,
+            assetId: fields.output?.coinAssetId,
+            to: fields.output?.coinTo
+        }, {
+            to: BYTES,
+            amount: BIG_NAT,
+            assetId: BYTES
+        })
+    })
+
+    let ContractOutput = object({
+        transactionIndex: NAT,
+        index: NAT,
+        ...project({
+            inputIndex: fields.output?.contractInputIndex,
+            balanceRoot: fields.output?.contractBalanceRoot,
+            stateRoot: fields.output?.contractStateRoot
+        }, {
+            inputIndex: NAT,
+            balanceRoot: BYTES,
+            stateRoot: BYTES
+        })
+    })
+
+    let ChangeOutput = object({
+        transactionIndex: NAT,
+        index: NAT,
+        ...project({
+            amount: fields.output?.changeAmount,
+            assetId: fields.output?.changeAssetId,
+            to: fields.output?.changeTo
+        }, {
+            to: BYTES,
+            amount: BIG_NAT,
+            assetId: BYTES
+        })
+    })
+
+    let VariableOutput = object({
+        transactionIndex: NAT,
+        index: NAT,
+        ...project({
+            to: fields.output?.variableTo,
+            amount: fields.output?.variableAmount,
+            assetId: fields.output?.variableAssetId
+        }, {
+            to: BYTES,
+            amount: BIG_NAT,
+            assetId: BYTES
+        })
+    })
+
+    let ContractCreated = object({
+        transactionIndex: NAT,
+        index: NAT,
+        ...project({
+            contract: fields.output?.contractCreatedContract,
+            stateRoot: fields.output?.contractCreatedStateRoot
+        }, {
+            contract: object({
+                id: BYTES,
+                bytecode: BYTES,
+                salt: BYTES
+            }),
+            stateRoot: BYTES
+        })
+    })
+
+    let Ouput = taggedUnion('type', {
+        CoinOutput,
+        ContractOutput,
+        ChangeOutput,
+        VariableOutput,
+        ContractCreated
+    })
+
+    return object({
+        header: BlockHeader,
+        transactions: option(array(Transaction)),
+        receipts: option(array(Receipt)),
+        inputs: option(array(Input)),
+        outputs: option(array(Ouput)),
+    })
+})
+
+
+function project<T>(fields: Selector<keyof T> | undefined, obj: T): Partial<T> {
+    if (fields == null) return {}
+    let result: Partial<T> = {}
+    let key: keyof T
+    for (key in obj) {
+        if (fields[key]) {
+            result[key] = obj[key]
+        }
+    }
+    return result
+}
+
+
+export function isEmpty(obj: object): boolean {
+    for (let _ in obj) {
+        return false
+    }
+    return true
+}

package/src/archive/source.ts

@@ -0,0 +1,88 @@
+import {BlockHeader} from '@subsquid/fuel-normalization'
+import {assertNotNull} from '@subsquid/util-internal'
+import {ArchiveClient} from '@subsquid/util-internal-archive-client'
+import {archiveIngest} from '@subsquid/util-internal-ingest-tools'
+import {getRequestAt, mapRangeRequestList, RangeRequestList} from '@subsquid/util-internal-range'
+import {array, cast} from '@subsquid/util-internal-validation'
+import assert from 'assert'
+import {DataRequest} from '../data/data-request'
+import {getDataSchema} from './data-schema'
+import {PartialBlock} from '../data/data-partial'
+
+
+export class FuelGateway {
+    constructor(private client: ArchiveClient) {}
+
+    getFinalizedHeight(): Promise<number> {
+        return this.client.getHeight()
+    }
+
+    async getBlockHash(height: number): Promise<string | undefined> {
+        let blocks = await this.client.query({
+            fromBlock: height,
+            toBlock: height,
+            includeAllBlocks: true
+        })
+        assert(blocks.length == 1)
+        return blocks[0].header.hash
+    }
+
+    async getBlockHeader(height: number): Promise<BlockHeader> {
+        let blocks = await this.client.query({
+            type: 'fuel',
+            fromBlock: height,
+            toBlock: height,
+            includeAllBlocks: true,
+            fields: {
+                block: {
+                    daHeight: true,
+                    transactionsRoot: true,
+                    transactionsCount: true,
+                    messageReceiptRoot: true,
+                    messageReceiptCount: true,
+                    prevRoot: true,
+                    time: true,
+                    applicationHash: true
+                }
+            }
+        })
+        assert(blocks.length == 1)
+        let {number, ...rest} = blocks[0].header
+        return {
+            height: number,
+            ...rest
+        } as BlockHeader
+    }
+
+    async *getBlockStream(requests: RangeRequestList<DataRequest>, stopOnHead?: boolean | undefined): AsyncIterable<PartialBlock[]> {
+        let archiveRequests = mapRangeRequestList(requests, req => {
+            let {fields, ...items} = req
+            return {
+                type: 'fuel',
+                fields,
+                ...items
+            }
+        })
+
+        for await (let batch of archiveIngest({
+            client: this.client,
+            requests: archiveRequests,
+            stopOnHead
+        })) {
+            let req = getRequestAt(requests, batch.blocks[0].header.number)
+
+            let blocks = cast(
+                array(getDataSchema(assertNotNull(req?.fields))),
+                batch.blocks
+            ).map(b => {
+                let {header: {number, ...hdr}, ...items} = b
+                return {
+                    header: {height: number, ...hdr},
+                    ...items
+                }
+            })
+
+            yield blocks
+        }
+    }
+}

package/src/data/data-partial.ts

@@ -0,0 +1,25 @@
+import type * as data from '@subsquid/fuel-data/lib/data'
+import type {MakePartial} from './util'
+
+
+export type BlockRequiredFields = 'height' | 'hash'
+export type TransactionRequiredFields = 'index'
+export type ReceiptRequiredFields = 'transactionIndex' | 'index'
+export type InputRequiredFields = 'transactionIndex' | 'index' | 'type'
+export type OutputRequiredFields = 'transactionIndex' | 'index' | 'type'
+
+
+export type PartialBlockHeader = MakePartial<data.BlockHeader, BlockRequiredFields>
+export type PartialTransaction = MakePartial<data.Transaction, TransactionRequiredFields>
+export type PartialReceipt = MakePartial<data.Receipt, ReceiptRequiredFields>
+export type PartialInput = MakePartial<data.TransactionInput, InputRequiredFields>
+export type PartialOutput = MakePartial<data.TransactionOutput, OutputRequiredFields>
+
+
+export interface PartialBlock {
+    header: PartialBlockHeader
+    transactions?: PartialTransaction[]
+    receipts?: PartialReceipt[]
+    inputs?: PartialInput[]
+    outputs?: PartialOutput[]
+}

package/src/data/data-request.ts

@@ -0,0 +1,44 @@
+import {ReceiptType, TransactionType, InputType, OutputType} from '@subsquid/fuel-data/lib/data'
+import {FieldSelection, Bytes} from './model'
+
+
+export interface DataRequest {
+    fields?: FieldSelection
+    includeAllBlocks?: boolean
+    transactions?: TransactionRequest[]
+    receipts?: ReceiptRequest[]
+    inputs?: InputRequest[]
+    outputs?: OutputRequest[]
+}
+
+
+export interface TransactionRequest {
+    type?: TransactionType[]
+    receipts?: boolean
+    inputs?: boolean
+    outputs?: boolean
+}
+
+
+export interface ReceiptRequest {
+    type?: ReceiptType[]
+    logDataContract?: Bytes[]
+    transaction?: boolean
+}
+
+
+export interface InputRequest {
+    type?: InputType[]
+    coinOwner?: Bytes[]
+    coinAssetId?: Bytes[]
+    contractContract?: Bytes[]
+    messageSender?: Bytes[]
+    messageRecipient?: Bytes[]
+    transaction?: boolean
+}
+
+
+export interface OutputRequest {
+    type?: OutputType[]
+    transaction?: boolean
+}

package/src/data/model.ts

@@ -0,0 +1,114 @@
+import type * as data from '@subsquid/fuel-data/lib/data'
+import type {
+    BlockRequiredFields,
+    ReceiptRequiredFields,
+    InputRequiredFields,
+    OutputRequiredFields,
+    TransactionRequiredFields
+} from './data-partial'
+import type {GetFields, Select, Selector, Simplify} from './util'
+
+
+/**
+ * Hex encoded binary string
+ */
+export type Bytes = string
+
+
+type AddPrefix<Prefix extends string, S extends string> = `${Prefix}${Capitalize<S>}`
+
+
+export interface FieldSelection {
+    block?: Selector<Exclude<keyof data.BlockHeader, BlockRequiredFields>>
+    transaction?: Selector<Exclude<keyof data.Transaction, TransactionRequiredFields>>
+    receipt?: Selector<Exclude<keyof data.Receipt, ReceiptRequiredFields>>
+    input?: Selector<
+        AddPrefix<'coin', Exclude<keyof data.InputCoin, InputRequiredFields>> |
+        AddPrefix<'contract', Exclude<keyof data.InputContract, InputRequiredFields>> |
+        AddPrefix<'message', Exclude<keyof data.InputMessage, InputRequiredFields>>
+    >
+    output?: Selector<
+        AddPrefix<'coin', Exclude<keyof data.CoinOutput, OutputRequiredFields>> |
+        AddPrefix<'contract', Exclude<keyof data.ContractOutput, OutputRequiredFields>> |
+        AddPrefix<'change', Exclude<keyof data.ChangeOutput, OutputRequiredFields>> |
+        AddPrefix<'variable', Exclude<keyof data.VariableOutput, OutputRequiredFields>> |
+        AddPrefix<'contractCreated', Exclude<keyof data.ContractCreated, OutputRequiredFields>>
+    >
+}
+
+
+export const DEFAULT_FIELDS = {
+    block: {
+        time: true
+    },
+    transaction: {
+        hash: true,
+        type: true,
+        status: true
+    },
+    receipt: {
+        receiptType: true
+    },
+    input: {},
+    output: {},
+} as const
+
+
+type Item<
+    Data,
+    RequiredFields extends keyof Data,
+    F extends FieldSelection,
+    K extends keyof FieldSelection
+> = Simplify<
+    Pick<Data, RequiredFields> &
+    Select<Data, GetFields<FieldSelection, typeof DEFAULT_FIELDS, F, K>>
+>
+
+
+export type BlockHeader<F extends FieldSelection = {}> = Item<
+    data.BlockHeader,
+    BlockRequiredFields,
+    F,
+    'block'
+>
+
+
+export type Transaction<F extends FieldSelection = {}> = Item<
+    data.Transaction,
+    TransactionRequiredFields,
+    F,
+    'transaction'
+>
+
+
+export type Receipt<F extends FieldSelection = {}> = Item<
+    data.Receipt,
+    ReceiptRequiredFields,
+    F,
+    'receipt'
+>
+
+
+export type Input<F extends FieldSelection = {}> = Item<
+    data.TransactionInput,
+    InputRequiredFields,
+    F,
+    'input'
+>
+
+
+export type Output<F extends FieldSelection = {}> = Item<
+    data.TransactionOutput,
+    OutputRequiredFields,
+    F,
+    'output'
+>
+
+
+export interface Block<F extends FieldSelection = {}> {
+    header: BlockHeader<F>
+    transactions: Transaction<F>[]
+    receipts: Receipt<F>[]
+    inputs: Input<F>[]
+    outputs: Output<F>[]
+}

package/src/data/util.ts

@@ -0,0 +1,42 @@
+export type Simplify<T> = {
+    [K in keyof T]: T[K]
+} & {}
+
+
+export type ExcludeUndefined<T> = {
+    [K in keyof T as undefined extends T[K] ? never : K]: T[K]
+} & {}
+
+
+export type GetFields<
+    FieldSelectionType,
+    Defaults extends FieldSelectionType,
+    Selection extends FieldSelectionType,
+    K extends keyof FieldSelectionType
+> = TrueFields<MergeDefault<Selection[K], Defaults[K]>>
+
+
+type MergeDefault<T, D> = Simplify<
+    undefined extends T ? D : Omit<D, keyof ExcludeUndefined<T>> & ExcludeUndefined<T>
+>
+
+
+type TrueFields<F> = keyof {
+    [K in keyof F as true extends F[K] ? K : never]: true
+}
+
+
+export type Select<T, Fields> = T extends any ? Simplify<Pick<T, Extract<keyof T, Fields>>> : never
+
+
+export type Selector<Fields extends string | number | symbol> = {
+    [P in Fields]?: boolean
+}
+
+
+export type MakePartial<T, Required extends keyof T> = Simplify<
+    Pick<T, Required> &
+    {
+        [K in keyof T as K extends Required ? never : K]+?: T[K]
+    }
+>

package/src/fields.ts

@@ -0,0 +1,35 @@
+import {DEFAULT_FIELDS, FieldSelection} from './data/model'
+import {Selector} from './data/util'
+
+
+/**
+ * Get effective set of selected fields.
+ */
+export function getFields(fields: FieldSelection | undefined): FieldSelection {
+    return {
+        block: merge(DEFAULT_FIELDS.block, fields?.block),
+        transaction: merge(DEFAULT_FIELDS.transaction, fields?.transaction),
+        receipt: merge(DEFAULT_FIELDS.receipt, fields?.receipt),
+        input: merge(DEFAULT_FIELDS.input, fields?.input),
+        output: merge(DEFAULT_FIELDS.output, fields?.output),
+    }
+}
+
+
+function merge<Keys extends string>(def: Selector<Keys>, requested: Selector<Keys> = {}): Selector<Keys> {
+    let fields: Selector<Keys> = {}
+
+    for (let key in def) {
+        if (requested[key] !== false) {
+            fields[key] = def[key]
+        }
+    }
+
+    for (let key in requested) {
+        if (requested[key]) {
+            fields[key] = true
+        }
+    }
+
+    return fields
+}