@subsquid/evm-stream 0.0.1-2-0.3887d2

package/src/schema.ts

@@ -0,0 +1,263 @@
+import {FieldSelection} from './query'
+import {
+    array,
+    BYTES,
+    NAT,
+    object,
+    oneOf,
+    option,
+    QTY,
+    SMALL_QTY,
+    STRING,
+    STRING_FLOAT,
+    taggedUnion,
+    withDefault,
+    withSentinel,
+} from '@subsquid/util-internal-validation'
+
+export function getBlockHeaderProps(fields: FieldSelection['block'], forArchive: boolean) {
+    let natural = forArchive ? NAT : SMALL_QTY
+    return {
+        number: natural,
+        hash: BYTES,
+        parentHash: BYTES,
+        ...project(fields, {
+            nonce: withSentinel('BlockHeader.nonce', '0x', BYTES),
+            sha3Uncles: withSentinel('BlockHeader.sha3Uncles', '0x', BYTES),
+            logsBloom: withSentinel('BlockHeader.logsBloom', '0x', BYTES),
+            transactionsRoot: withSentinel('BlockHeader.transactionsRoot', '0x', BYTES),
+            stateRoot: withSentinel('BlockHeader.stateRoot', '0x', BYTES),
+            receiptsRoot: withSentinel('BlockHeader.receiptsRoot', '0x', BYTES),
+            mixHash: withSentinel('BlockHeader.mixHash', '0x', BYTES),
+            miner: withSentinel('BlockHeader.miner', '0x', BYTES),
+            difficulty: withSentinel('BlockHeader.difficulty', -1n, QTY),
+            totalDifficulty: withSentinel('BlockHeader.totalDifficulty', -1n, QTY),
+            extraData: withSentinel('BlockHeader.extraData', '0x', BYTES),
+            size: withSentinel('BlockHeader.size', -1, natural),
+            gasLimit: withSentinel('BlockHeader.gasLimit', -1n, QTY),
+            gasUsed: withSentinel('BlockHeader.gasUsed', -1n, QTY),
+            baseFeePerGas: withSentinel('BlockHeader.baseFeePerGas', -1n, QTY),
+            timestamp: withSentinel('BlockHeader.timestamp', 0, natural),
+            l1BlockNumber: withDefault(0, natural),
+        }),
+    }
+}
+
+export function getTxProps(fields: FieldSelection['transaction'], forArchive: boolean) {
+    let natural = forArchive ? NAT : SMALL_QTY
+
+    let Authorization = object({
+        chainId: natural,
+        nonce: natural,
+        address: BYTES,
+        yParity: natural,
+        r: BYTES,
+        s: BYTES,
+    })
+
+    return {
+        transactionIndex: natural,
+        ...project(fields, {
+            hash: BYTES,
+            from: BYTES,
+            to: option(BYTES),
+            gas: withSentinel('Transaction.gas', -1n, QTY),
+            gasPrice: withSentinel('Transaction.gasPrice', -1n, QTY),
+            maxFeePerGas: option(QTY),
+            maxPriorityFeePerGas: option(QTY),
+            input: BYTES,
+            nonce: withSentinel('Transaction.nonce', -1, natural),
+            value: withSentinel('Transaction.value', -1n, QTY),
+            v: withSentinel('Transaction.v', -1n, QTY),
+            r: withSentinel('Transaction.r', '0x', BYTES),
+            s: withSentinel('Transaction.s', '0x', BYTES),
+            yParity: option(natural),
+            chainId: option(natural),
+            authorizationList: option(array(Authorization)),
+        }),
+    }
+}
+
+export function getTxReceiptProps(fields: FieldSelection['transaction'], forArchive: boolean) {
+    let natural = forArchive ? NAT : SMALL_QTY
+    return project(fields, {
+        gasUsed: withSentinel('Receipt.gasUsed', -1n, QTY),
+        cumulativeGasUsed: withSentinel('Receipt.cumulativeGasUsed', -1n, QTY),
+        effectiveGasPrice: withSentinel('Receipt.effectiveGasPrice', -1n, QTY),
+        contractAddress: option(BYTES),
+        type: withSentinel('Receipt.type', -1, natural),
+        status: withSentinel('Receipt.status', -1, natural),
+        l1Fee: option(QTY),
+        l1FeeScalar: option(STRING_FLOAT),
+        l1GasPrice: option(QTY),
+        l1GasUsed: option(QTY),
+        l1BlobBaseFee: option(QTY),
+        l1BlobBaseFeeScalar: option(natural),
+        l1BaseFeeScalar: option(natural),
+    })
+}
+
+export function getLogProps(fields: FieldSelection['log'], forArchive: boolean) {
+    let natural = forArchive ? NAT : SMALL_QTY
+    return {
+        logIndex: natural,
+        transactionIndex: natural,
+        ...project(fields, {
+            transactionHash: BYTES,
+            address: BYTES,
+            data: BYTES,
+            topics: array(BYTES),
+        }),
+    }
+}
+
+export function getTraceFrameValidator(fields: FieldSelection['trace'], forArchive: boolean) {
+    let traceBase = {
+        transactionIndex: forArchive ? NAT : undefined,
+        traceAddress: array(NAT),
+        ...project(fields, {
+            subtraces: NAT,
+            error: option(STRING),
+            revertReason: option(STRING),
+        }),
+    }
+
+    let traceCreateAction = project(
+        {
+            from: fields?.createFrom || !forArchive,
+            value: fields?.createValue,
+            gas: fields?.createGas,
+            init: fields?.createInit,
+        },
+        {
+            from: BYTES,
+            value: QTY,
+            gas: QTY,
+            init: withDefault('0x', BYTES),
+        }
+    )
+
+    let traceCreateResult = project(
+        {
+            gasUsed: fields?.createResultGasUsed,
+            code: fields?.createResultCode,
+            address: fields?.createResultAddress,
+        },
+        {
+            gasUsed: QTY,
+            code: withDefault('0x', BYTES),
+            address: withDefault('0x0000000000000000000000000000000000000000', BYTES),
+        }
+    )
+
+    let TraceCreate = object({
+        ...traceBase,
+        action: isEmpty(traceCreateAction) ? undefined : object(traceCreateAction),
+        result: isEmpty(traceCreateResult) ? undefined : option(object(traceCreateResult)),
+    })
+
+    let traceCallAction = project(
+        {
+            callType: fields?.callCallType,
+            from: forArchive ? fields?.callFrom : true,
+            to: forArchive ? fields?.callTo : true,
+            value: fields?.callValue,
+            gas: fields?.callGas,
+            input: forArchive ? fields?.callInput : true,
+            sighash: forArchive ? fields?.callSighash : false,
+        },
+        {
+            callType: STRING,
+            from: BYTES,
+            to: BYTES,
+            value: option(QTY),
+            gas: QTY,
+            input: BYTES,
+            sighash: withDefault('0x', BYTES),
+        }
+    )
+
+    let traceCallResult = project(
+        {
+            gasUsed: fields?.callResultGasUsed,
+            output: fields?.callResultOutput,
+        },
+        {
+            gasUsed: QTY,
+            output: withDefault('0x', BYTES),
+        }
+    )
+
+    let TraceCall = object({
+        ...traceBase,
+        action: isEmpty(traceCallAction) ? undefined : object(traceCallAction),
+        result: isEmpty(traceCallResult) ? undefined : option(object(traceCallResult)),
+    })
+
+    let traceSuicideAction = project(
+        {
+            address: fields?.suicideAddress,
+            refundAddress: forArchive ? fields?.suicideRefundAddress : true,
+            balance: fields?.suicideBalance,
+        },
+        {
+            address: BYTES,
+            refundAddress: BYTES,
+            balance: QTY,
+        }
+    )
+
+    let TraceSuicide = object({
+        ...traceBase,
+        action: isEmpty(traceSuicideAction) ? undefined : object(traceSuicideAction),
+    })
+
+    let traceRewardAction = project(
+        {
+            author: forArchive ? fields?.rewardAuthor : true,
+            value: fields?.rewardValue,
+            type: fields?.rewardType,
+        },
+        {
+            author: BYTES,
+            value: QTY,
+            type: STRING,
+        }
+    )
+
+    let TraceReward = object({
+        ...traceBase,
+        action: isEmpty(traceRewardAction) ? undefined : object(traceRewardAction),
+    })
+
+    return taggedUnion('type', {
+        create: TraceCreate,
+        call: TraceCall,
+        suicide: TraceSuicide,
+        reward: TraceReward,
+    })
+}
+
+export function project<T extends object, F extends {[K in keyof T]?: boolean}>(
+    fields: F | undefined,
+    obj: T
+): Partial<T> {
+    if (fields == null) return {}
+    let result: Partial<T> = {}
+    let key: keyof T
+    for (key in obj) {
+        if (fields[key]) {
+            result[key] = obj[key]
+        }
+    }
+    return result
+}
+
+export function isEmpty(obj: object): boolean {
+    for (let _ in obj) {
+        return false
+    }
+    return true
+}
+
+export function assertAssignable<A, B extends A>(): void {}

package/src/source.ts

@@ -0,0 +1,261 @@
+import {applyRangeBound, mergeRangeRequests, Range, RangeRequest} from '@subsquid/util-internal-range'
+import {PortalClient, PortalStreamData} from '@subsquid/portal-client'
+import {weakMemo} from '@subsquid/util-internal'
+import {array, BYTES, cast, NAT, object, STRING, taggedUnion, withDefault} from '@subsquid/util-internal-validation'
+import {
+    getBlockHeaderProps,
+    getTxProps,
+    getTxReceiptProps,
+    getLogProps,
+    getTraceFrameValidator,
+    project,
+} from './schema'
+import {
+    BlockData,
+    DataRequest,
+    EvmQueryOptions,
+    FieldSelection,
+    mergeDataRequests,
+    MergeFieldSelection,
+    mergeSelection,
+    Response,
+} from './query'
+import {DataSource, DataSourceStream, DataSourceStreamData} from '@subsquid/data-source'
+
+export interface EvmPortalDataSourceOptions<Q extends EvmQueryOptions> {
+    portal: string | PortalClient
+    query: Q
+}
+
+export class EvmPortalDataSource<
+    Q extends EvmQueryOptions,
+    B extends BlockData<GetFields<Q['fields']>> = BlockData<GetFields<Q['fields']>>
+> implements DataSource<B>
+{
+    private portal: PortalClient
+    private fields: Q['fields']
+    private requests: RangeRequest<DataRequest>[]
+
+    constructor(options: EvmPortalDataSourceOptions<Q>) {
+        this.portal = typeof options.portal === 'string' ? new PortalClient({url: options.portal}) : options.portal
+        this.fields = options.query.fields
+        this.requests = mergeRangeRequests(options.query.requests, mergeDataRequests)
+    }
+
+    getHeight(): Promise<number> {
+        return this.portal.getFinalizedHeight()
+    }
+
+    getFinalizedHeight(): Promise<number> {
+        return this.portal.getFinalizedHeight()
+    }
+
+    getBlockStream(range?: Range, stopOnHead?: boolean): DataSourceStream<B> {
+        let fields = getFields(this.fields)
+        let requests = applyRangeBound(this.requests, range)
+
+        let {writable, readable} = new TransformStream<
+            PortalStreamData<BlockData<typeof fields>>,
+            DataSourceStreamData<B>
+        >({
+            transform: async (data, controller) => {
+                let blocks = data.map((b) => {
+                    let block = mapBlock(b, fields)
+                    Object.defineProperty(block, DataSource.blockRef, {
+                        value: {hash: block.header.hash, number: block.header.number},
+                    })
+                    return block
+                })
+
+                Object.defineProperty(blocks, DataSource.finalizedHead, {
+                    value: data[PortalClient.finalizedHead],
+                })
+
+                controller.enqueue(blocks as DataSourceStreamData<B>)
+            },
+        })
+
+        const ingest = async () => {
+            for (let request of requests) {
+                let query = {
+                    type: 'evm',
+                    fromBlock: request.range.from,
+                    toBlock: request.range.to,
+                    fields,
+                    ...request.request,
+                }
+
+                await this.portal.getFinalizedStream(query, {stopOnHead}).pipeTo(writable, {
+                    preventClose: true,
+                })
+            }
+        }
+
+        ingest()
+            .then(
+                () => writable.close(),
+                (reason) => writable.abort(reason)
+            )
+            .catch(() => {})
+
+        return readable
+    }
+}
+
+export const getBlockValidator = weakMemo(<F extends FieldSelection>(fields: F) => {
+    let BlockHeader = object(getBlockHeaderProps(fields.block, true))
+
+    let Transaction = object({
+        hash: fields.transaction?.hash ? BYTES : undefined,
+        ...getTxProps(fields.transaction, true),
+        sighash: fields.transaction?.sighash ? withDefault('0x', BYTES) : undefined,
+        ...getTxReceiptProps(fields.transaction, true),
+    })
+
+    let Log = object(getLogProps(fields.log, true))
+
+    let Trace = getTraceFrameValidator(fields.trace, true)
+
+    let stateDiffBase = {
+        transactionIndex: NAT,
+        address: BYTES,
+        key: STRING,
+    }
+
+    let StateDiff = taggedUnion('kind', {
+        ['=']: object({...stateDiffBase}),
+        ['+']: object({...stateDiffBase, ...project(fields.stateDiff, {next: BYTES})}),
+        ['*']: object({...stateDiffBase, ...project(fields.stateDiff, {prev: BYTES, next: BYTES})}),
+        ['-']: object({...stateDiffBase, ...project(fields.stateDiff, {prev: BYTES})}),
+    })
+
+    return object({
+        header: BlockHeader,
+        transactions: withDefault([], array(Transaction)),
+        logs: withDefault([], array(Log)),
+        traces: withDefault([], array(Trace)),
+        stateDiffs: withDefault([], array(StateDiff)),
+    })
+})
+
+export function mapBlock<F extends FieldSelection, B extends BlockData<F> = BlockData<F>>(
+    rawBlock: unknown,
+    fields: F
+): B {
+    let validator = getBlockValidator(fields)
+    let block = cast(validator, rawBlock)
+
+    // let {number, hash, parentHash, ...hdr} = src.header
+    // if (hdr.timestamp) {
+    //     hdr.timestamp = hdr.timestamp * 1000 // convert to ms
+    // }
+
+    // let header = new BlockHeader(number, hash, parentHash)
+    // Object.assign(header, hdr)
+
+    // let block = new Block(header)
+
+    // if (src.transactions) {
+    //     for (let {transactionIndex, ...props} of src.transactions) {
+    //         let tx = new Transaction(header, transactionIndex)
+    //         Object.assign(tx, props)
+    //         block.transactions.push(tx)
+    //     }
+    // }
+
+    // if (src.logs) {
+    //     for (let {logIndex, transactionIndex, ...props} of src.logs) {
+    //         let log = new Log(header, logIndex, transactionIndex)
+    //         Object.assign(log, props)
+    //         block.logs.push(log)
+    //     }
+    // }
+
+    // if (src.traces) {
+    //     for (let {transactionIndex, traceAddress, type, ...props} of src.traces) {
+    //         transactionIndex = assertNotNull(transactionIndex)
+    //         let trace: Trace
+    //         switch (type) {
+    //             case 'create':
+    //                 trace = new TraceCreate(header, transactionIndex, traceAddress)
+    //                 break
+    //             case 'call':
+    //                 trace = new TraceCall(header, transactionIndex, traceAddress)
+    //                 break
+    //             case 'suicide':
+    //                 trace = new TraceSuicide(header, transactionIndex, traceAddress)
+    //                 break
+    //             case 'reward':
+    //                 trace = new TraceReward(header, transactionIndex, traceAddress)
+    //                 break
+    //             default:
+    //                 throw unexpectedCase()
+    //         }
+    //         Object.assign(trace, props)
+    //         block.traces.push(trace)
+    //     }
+    // }
+
+    // if (src.stateDiffs) {
+    //     for (let {transactionIndex, address, key, kind, ...props} of src.stateDiffs) {
+    //         let diff: StateDiff
+    //         switch (kind) {
+    //             case '=':
+    //                 diff = new StateDiffNoChange(header, transactionIndex, address, key)
+    //                 break
+    //             case '+':
+    //                 diff = new StateDiffAdd(header, transactionIndex, address, key)
+    //                 break
+    //             case '*':
+    //                 diff = new StateDiffChange(header, transactionIndex, address, key)
+    //                 break
+    //             case '-':
+    //                 diff = new StateDiffDelete(header, transactionIndex, address, key)
+    //                 break
+    //             default:
+    //                 throw unexpectedCase()
+    //         }
+    //         Object.assign(diff, props)
+    //         block.stateDiffs.push(diff)
+    //     }
+    // }
+
+    // setUpRelations(block)
+
+    return block as unknown as B
+}
+
+function getFields<T extends FieldSelection>(fields: T): GetFields<T> {
+    return mergeSelection(REQUIRED_FIELDS, fields)
+}
+
+type GetFields<F extends FieldSelection> = MergeFieldSelection<ReqiredFieldSelection, F>
+
+// type A = BlockData<GetFields<{}>>['header']
+
+type ReqiredFieldSelection = typeof REQUIRED_FIELDS
+
+const REQUIRED_FIELDS = {
+    block: {
+        number: true,
+        hash: true,
+        parentHash: true,
+    },
+    // transaction: {
+    //     transactionIndex: true,
+    // },
+    // log: {
+    //     logIndex: true,
+    //     transactionIndex: true,
+    // },
+    // trace: {
+    //     transactionIndex: true,
+    //     traceAddress: true,
+    //     type: true,
+    // },
+    // stateDiff: {
+    //     transactionIndex: true,
+    //     address: true,
+    //     key: true,
+    // },
+} as const satisfies FieldSelection