@subsquid/fuel-stream 0.0.1

package/src/filter.ts

@@ -0,0 +1,229 @@
+import {EntityFilter, FilterBuilder} from '@subsquid/util-internal-processor-tools'
+import {assertNotNull, groupBy, weakMemo} from '@subsquid/util-internal'
+import {getRequestAt, RangeRequest} from '@subsquid/util-internal-range'
+import {Block, TransactionInput, TransactionOutput, Receipt, Transaction} from '@subsquid/fuel-normalization'
+import {DataRequest} from './data/data-request'
+
+
+class IncludeSet {
+    public readonly receipts = new Set<Receipt>()
+    public readonly transactions = new Set<Transaction>()
+    public readonly inputs = new Set<TransactionInput>()
+    public readonly outputs = new Set<TransactionOutput>()
+
+    addReceipt(receipt?: Receipt): void {
+        if (receipt) {
+            this.receipts.add(receipt)
+        }
+    }
+
+    addTransaction(tx?: Transaction): void {
+        if (tx) {
+            this.transactions.add(tx)
+        }
+    }
+
+    addInput(input?: TransactionInput): void {
+        if (input) {
+            this.inputs.add(input)
+        }
+    }
+
+    addOutput(output?: TransactionOutput): void {
+        if (output) {
+            this.outputs.add(output)
+        }
+    }
+}
+
+
+interface ReceiptRelations {
+    transaction?: boolean
+}
+
+
+interface TransactionRelations {
+    receipts?: boolean
+    inputs?: boolean
+    outputs?: boolean
+}
+
+
+interface InputRelations {
+    transaction?: boolean
+}
+
+
+interface OutputRelations {
+    transaction?: boolean
+}
+
+
+function buildReceiptFilter(dataRequest: DataRequest): EntityFilter<Receipt, ReceiptRelations> {
+    let receipts = new EntityFilter<Receipt, ReceiptRelations>()
+
+    dataRequest.receipts?.forEach(req => {
+        let {type, logDataContract, ...relations} = req
+        let filter = new FilterBuilder<Receipt>()
+        filter.propIn('receiptType', type)
+        filter.propIn('contract', logDataContract)
+        receipts.add(filter, relations)
+    })
+
+    return receipts
+}
+
+
+function buildTransactionFilter(dataRequest: DataRequest): EntityFilter<Transaction, TransactionRelations> {
+    let transactions = new EntityFilter<Transaction, TransactionRelations>()
+
+    dataRequest.transactions?.forEach(req => {
+        let {type, ...relations} = req
+        let filter = new FilterBuilder<Transaction>()
+        filter.propIn('type', type)
+        transactions.add(filter, relations)
+    })
+
+    return transactions
+}
+
+
+function buildInputFilter(dataRequest: DataRequest): EntityFilter<TransactionInput, InputRelations> {
+    let inputs = new EntityFilter<TransactionInput, InputRelations>()
+
+    dataRequest.inputs?.forEach(req => {
+        let {
+            type,
+            coinOwner,
+            coinAssetId,
+            contractContract,
+            messageSender,
+            messageRecipient,
+            ...relations
+        } = req
+        let filter = new FilterBuilder<TransactionInput>()
+        filter.propIn('type', req.type)
+        filter.getIn(input => input.type == 'InputCoin' && assertNotNull(input.owner), coinOwner)
+        filter.getIn(input => input.type == 'InputCoin' && assertNotNull(input.assetId), coinAssetId)
+        filter.getIn(input => input.type == 'InputContract' && assertNotNull(input.contract), contractContract)
+        filter.getIn(input => input.type == 'InputMessage' && assertNotNull(input.sender), messageSender)
+        filter.getIn(input => input.type == 'InputMessage' && assertNotNull(input.recipient), messageRecipient)
+        inputs.add(filter, relations)
+    })
+
+    return inputs
+}
+
+
+function buildOutputFilter(dataRequest: DataRequest): EntityFilter<TransactionOutput, OutputRelations> {
+    let outputs = new EntityFilter<TransactionOutput, OutputRelations>()
+
+    dataRequest.outputs?.forEach(req => {
+        let {type, ...relations} = req
+        let filter = new FilterBuilder<TransactionOutput>()
+        filter.propIn('type', type)
+        outputs.add(filter, relations)
+    })
+
+    return outputs
+}
+
+
+const getItemFilter = weakMemo((dataRequest: DataRequest) => {
+    return {
+        receipts: buildReceiptFilter(dataRequest),
+        transactions: buildTransactionFilter(dataRequest),
+        inputs: buildInputFilter(dataRequest),
+        outputs: buildOutputFilter(dataRequest),
+    }
+})
+
+
+export function filterBlock(block: Block, dataRequest: DataRequest): void {
+    let items = getItemFilter(dataRequest)
+
+    let include = new IncludeSet()
+
+    let transactions = new Map(block.transactions.map(tx => [tx.index, tx]))
+    let inputsByTx = groupBy(block.inputs, input => input.transactionIndex)
+    let outputsByTx = groupBy(block.outputs, ouput => ouput.transactionIndex)
+    let receiptsByTx = groupBy(block.receipts, receipt => receipt.transactionIndex)
+
+    if (items.receipts.present()) {
+        for (let receipt of block.receipts) {
+            let rel = items.receipts.match(receipt)
+            if (rel == null) continue
+            include.addReceipt(receipt)
+            if (rel.transaction) {
+                let tx = assertNotNull(transactions.get(receipt.transactionIndex))
+                include.addTransaction(tx)
+            }
+        }
+    }
+
+    if (items.transactions.present()) {
+        for (let tx of block.transactions) {
+            let rel = items.transactions.match(tx)
+            if (rel == null) continue
+            include.addTransaction(tx)
+            if (rel.receipts) {
+                let receipts = assertNotNull(receiptsByTx.get(tx.index))
+                for (let receipt of receipts) {
+                    include.addReceipt(receipt)
+                }
+            }
+            if (rel.inputs) {
+                let inputs = assertNotNull(inputsByTx.get(tx.index))
+                for (let input of inputs) {
+                    include.addInput(input)
+                }
+            }
+            if (rel.outputs) {
+                let outputs = assertNotNull(outputsByTx.get(tx.index))
+                for (let output of outputs) {
+                    include.addOutput(output)
+                }
+            }
+        }
+    }
+
+    if (items.inputs.present()) {
+        for (let input of block.inputs) {
+            let rel = items.inputs.match(input)
+            if (rel == null) continue
+            include.addInput(input)
+            if (rel.transaction) {
+                let tx = assertNotNull(transactions.get(input.transactionIndex))
+                include.addTransaction(tx)
+            }
+        }
+    }
+
+    if (items.outputs.present()) {
+        for (let output of block.outputs) {
+            let rel = items.outputs.match(output)
+            if (rel == null) continue
+            include.addOutput(output)
+            if (rel.transaction) {
+                let tx = assertNotNull(transactions.get(output.transactionIndex))
+                include.addTransaction(tx)
+            }
+        }
+    }
+
+    block.receipts = block.receipts.filter(receipt => include.receipts.has(receipt))
+    block.transactions = block.transactions.filter(tx => include.transactions.has(tx))
+    block.inputs = block.inputs.filter(input => include.inputs.has(input))
+    block.outputs = block.outputs.filter(output => include.outputs.has(output))
+}
+
+
+export function filterBlockBatch(requests: RangeRequest<DataRequest>[], blocks: Block[]): void {
+    for (let block of blocks) {
+        let dataRequest = getRequestAt(requests, block.header.height) || NO_DATA_REQUEST
+        filterBlock(block, dataRequest)
+    }
+}
+
+
+const NO_DATA_REQUEST: DataRequest = {}

package/src/graphql.ts

@@ -0,0 +1,51 @@
+import {mapRangeRequestList, RangeRequestList} from '@subsquid/util-internal-range'
+import {HttpDataSource} from '@subsquid/fuel-data/lib/data-source'
+import {BlockHeader, DataRequest as RawDataRequest} from '@subsquid/fuel-data/lib/raw-data'
+import {mapRawBlock} from '@subsquid/fuel-normalization'
+import {DataRequest} from './data/data-request'
+import {PartialBlock} from './data/data-partial'
+import {filterBlockBatch} from './filter'
+
+
+export class GraphqlDataSource {
+    constructor(private baseDataSource: HttpDataSource) { }
+
+    async getFinalizedHeight(): Promise<number> {
+        return this.baseDataSource.getFinalizedHeight()
+    }
+
+    getBlockHash(height: number): Promise<string | undefined> {
+        return this.baseDataSource.getBlockHash(height)
+    }
+
+    getBlockHeader(height: number): Promise<BlockHeader | undefined> {
+        return this.baseDataSource.getBlockHeader(height)
+    }
+
+    async *getBlockStream(
+        requests: RangeRequestList<DataRequest>,
+        stopOnHead?: boolean
+    ): AsyncIterable<PartialBlock[]> {
+        for await (let batch of this.baseDataSource.getFinalizedBlocks(
+            mapRangeRequestList(requests, toRawDataRequest),
+            stopOnHead
+        )) {
+            let blocks = batch.blocks.map(b => mapRawBlock(b))
+            filterBlockBatch(requests, blocks)
+            yield blocks
+        }
+    }
+}
+
+
+function toRawDataRequest(req: DataRequest): RawDataRequest {
+    return {
+        transactions: !!req.transactions?.length
+            || !!req.inputs?.length
+            || !!req.outputs?.length
+            || !!req.receipts?.length,
+        inputs: !!req.inputs?.length,
+        outputs: !!req.outputs?.length,
+        receipts: !!req.receipts?.length
+    }
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export * from './data/model'
+export * from './source'

package/src/source.ts

@@ -0,0 +1,416 @@
+import {BlockHeader} from '@subsquid/fuel-normalization'
+import {HttpAgent, HttpClient} from '@subsquid/http-client'
+import {Logger} from '@subsquid/logger'
+import {def, addErrorContext, last} from '@subsquid/util-internal'
+import {ArchiveClient} from '@subsquid/util-internal-archive-client'
+import {getOrGenerateSquidId} from '@subsquid/util-internal-processor-tools'
+import {
+    applyRangeBound,
+    mergeRangeRequests,
+    getSize,
+    Range,
+    RangeRequest,
+    RangeRequestList,
+    FiniteRange
+} from '@subsquid/util-internal-range'
+import {BlockHeader as RawBlockHeader} from '@subsquid/fuel-data/lib/raw-data'
+import {HttpDataSource} from '@subsquid/fuel-data/lib/data-source'
+import assert from 'assert'
+import {FuelGateway} from './archive/source'
+import {getFields} from './fields'
+import {Block, FieldSelection} from './data/model'
+import {GraphqlDataSource} from './graphql'
+import {PartialBlock} from './data/data-partial'
+import {
+    DataRequest,
+    ReceiptRequest,
+    InputRequest,
+    OutputRequest,
+    TransactionRequest
+} from './data/data-request'
+
+
+export interface GraphqlSettings {
+    /**
+     * GraphQL endpoint URL
+     */
+    url: string
+    /**
+     * Maximum number of concurrent `blocks` queries.
+     *
+     * Default is `10`
+     */
+    strideConcurrency?: number
+    /**
+     * `blocks` query size.
+     *
+     * Default is `5`.
+     */
+    strideSize?: number
+}
+
+
+export interface GatewaySettings {
+    /**
+     * Subsquid Network Gateway url
+     */
+    url: string
+    /**
+     * Request timeout in ms
+     */
+    requestTimeout?: number
+}
+
+
+interface BlockRange {
+    range?: Range
+}
+
+
+
+/**
+ * API and data that is passed to the data handler
+ */
+export interface DataHandlerContext<Store, F extends FieldSelection = {}> {
+    /**
+     * An instance of a structured logger.
+     */
+    log: Logger
+    /**
+     * Storage interface provided by the database
+     */
+    store: Store
+    /**
+     * List of blocks to map and process
+     */
+    blocks: Block<F>[]
+    /**
+     * Signals, that the processor reached the head of a chain.
+     *
+     * The head block is always included in `.blocks`.
+     */
+    isHead: boolean
+}
+
+
+export type FuelBatchProcessorFields<T> = T extends DataSourceBuilder<infer F> ? F : never
+
+
+export class DataSourceBuilder<F extends FieldSelection = {}> {
+    private requests: RangeRequest<DataRequest>[] = []
+    private fields?: FieldSelection
+    private blockRange?: Range
+    private gateway?: GatewaySettings
+    private graphql?: GraphqlSettings
+    private running = false
+
+    private assertNotRunning(): void {
+        if (this.running) {
+            throw new Error('Settings modifications are not allowed after start of processing')
+        }
+    }
+
+    /**
+     * Set Subsquid Network Gateway endpoint (ex Archive).
+     *
+     * Subsquid Network allows to get data from finalized blocks up to
+     * infinite times faster and more efficient than via regular GraphQL.
+     *
+     * @example
+     * processor.setGateway('https://v2.archive.subsquid.io/network/fuel-mainnet')
+     */
+    setGateway(url: string | GatewaySettings): this {
+        this.assertNotRunning()
+        if (typeof url == 'string') {
+            this.gateway = {url}
+        } else {
+            this.gateway = url
+        }
+        return this
+    }
+
+    /**
+     * Set up GraphQL data ingestion
+     */
+    setGraphql(settings?: GraphqlSettings): this {
+        this.assertNotRunning()
+        this.graphql = settings
+        return this
+    }
+
+    /**
+     * Limits the range of blocks to be processed.
+     *
+     * When the upper bound is specified,
+     * the processor will terminate with exit code 0 once it reaches it.
+     */
+    setBlockRange(range?: Range): this {
+        this.assertNotRunning()
+        this.blockRange = range
+        return this
+    }
+
+    /**
+     * Configure a set of fetched fields
+     */
+    setFields<F extends FieldSelection>(fields: F): DataSourceBuilder<F> {
+        this.assertNotRunning()
+        this.fields = fields
+        return this as any
+    }
+
+    private add(range: Range | undefined, request: DataRequest): void {
+        this.requests.push({
+            range: range || {from: 0},
+            request
+        })
+    }
+
+    /**
+     * By default, the processor will fetch only blocks
+     * which contain requested items. This method
+     * modifies such behaviour to fetch all chain blocks.
+     *
+     * Optionally a range of blocks can be specified
+     * for which the setting should be effective.
+     */
+    includeAllBlocks(range?: Range): this {
+        this.assertNotRunning()
+        this.add(range, {includeAllBlocks: true})
+        return this
+    }
+
+    addTransaction(options: TransactionRequest & BlockRange): this {
+        this.assertNotRunning()
+        let {range, ...req} = options
+        this.add(range, {
+            transactions: [req]
+        })
+        return this
+    }
+
+    addReceipt(options: ReceiptRequest & BlockRange): this {
+        this.assertNotRunning()
+        let {range, ...req} = options
+        this.add(range, {
+            receipts: [req]
+        })
+        return this
+    }
+
+    addInput(options: InputRequest & BlockRange): this {
+        this.assertNotRunning()
+        let {range, ...req} = options
+        this.add(range, {
+            inputs: [req]
+        })
+        return this
+    }
+
+    addOutput(options: OutputRequest & BlockRange): this {
+        this.assertNotRunning()
+        let {range, ...req} = options
+        this.add(range, {
+            outputs: [req]
+        })
+        return this
+    }
+
+    @def
+    private getRequests(): RangeRequest<DataRequest>[] {
+        function concat<T>(a?: T[], b?: T[]): T[] | undefined {
+            let result: T[] = []
+            if (a) {
+                result.push(...a)
+            }
+            if (b) {
+                result.push(...b)
+            }
+            return result.length == 0 ? undefined : result
+        }
+
+        let requests = mergeRangeRequests(this.requests, (a, b) => {
+            return {
+                includeAllBlocks: a.includeAllBlocks || b.includeAllBlocks,
+                transactions: concat(a.transactions, b.transactions),
+                receipts: concat(a.receipts, b.receipts),
+                inputs: concat(a.inputs, b.inputs),
+                outputs: concat(a.outputs, b.outputs),
+            }
+        })
+
+        let fields = getFields(this.fields)
+
+        requests = requests.map(({range, request}) => {
+            return {
+                range,
+                request: {
+                    fields,
+                    ...request
+                }
+            }
+        })
+
+        return applyRangeBound(requests, this.blockRange)
+    }
+
+    build(): DataSource<Block<F>> {
+        return new FuelDataSource(
+            this.getRequests(),
+            this.gateway,
+            this.graphql
+        ) as DataSource<Block<F>>
+    }
+}
+
+
+export interface DataSource<Block> {
+    getFinalizedHeight(): Promise<number>
+    getBlockHash(height: number): Promise<string | undefined>
+    getBlocksCountInRange(range: FiniteRange): number
+    getBlockStream(fromBlockHeight?: number): AsyncIterable<Block[]>
+}
+
+
+export type GetDataSourceBlock<T> = T extends DataSource<infer B> ? B : never
+
+
+class FuelDataSource implements DataSource<PartialBlock> {
+    private graphql?: GraphqlDataSource
+    private isConsistent?: boolean
+    private ranges: Range[]
+
+    constructor(
+        private requests: RangeRequestList<DataRequest>,
+        private gatewaySettings?: GatewaySettings,
+        graphqlSettings?: GraphqlSettings
+    ) {
+        assert(this.gatewaySettings || graphqlSettings, 'either gateway or GraphQL should be provided')
+        if (graphqlSettings) {
+            this.graphql = this.createGraphqlDataSource(graphqlSettings)
+        }
+        this.ranges = this.requests.map(req => req.range)
+    }
+
+    getFinalizedHeight(): Promise<number> {
+        if (this.graphql) {
+            return this.graphql.getFinalizedHeight()
+        } else {
+            return this.createGateway().getFinalizedHeight()
+        }
+    }
+
+    async getBlockHash(height: number): Promise<string | undefined> {
+        await this.assertConsistency()
+        if (this.gatewaySettings == null) {
+            assert(this.graphql)
+            return this.graphql.getBlockHash(height)
+        } else {
+            let gateway = this.createGateway()
+            let head = await gateway.getFinalizedHeight()
+            if (head >= height) return gateway.getBlockHash(height)
+            if (this.graphql) return this.graphql.getBlockHash(height)
+        }
+    }
+
+    private async assertConsistency(): Promise<void> {
+        if (this.isConsistent || this.gatewaySettings == null || this.graphql == null) return
+        let blocks = await this.performConsistencyCheck().catch(err => {
+            throw addErrorContext(
+                new Error(`Failed to check consistency between Subsquid Gateway and GraphQL endpoints`),
+                {reason: err}
+            )
+        })
+        if (blocks == null) {
+            this.isConsistent = true
+        } else {
+            throw addErrorContext(
+                new Error(`Provided Subsquid Gateway and GraphQL endpoints don't agree on block №${blocks.archiveBlock.height}`),
+                blocks
+            )
+        }
+    }
+
+    private async performConsistencyCheck(): Promise<{
+        archiveBlock: BlockHeader
+        gqlBlock: RawBlockHeader | null
+    } | undefined> {
+        let archive = this.createGateway()
+        let height = await archive.getFinalizedHeight()
+        let archiveBlock = await archive.getBlockHeader(height)
+        let gqlBlock = await this.graphql!.getBlockHeader(archiveBlock.height)
+        if (gqlBlock?.id === archiveBlock.hash && Number(gqlBlock.height) === archiveBlock.height) return
+        return {archiveBlock, gqlBlock: gqlBlock || null}
+    }
+
+    getBlocksCountInRange(range: FiniteRange): number {
+        return getSize(this.ranges, range)
+    }
+
+    async *getBlockStream(fromBlockHeight?: number): AsyncIterable<PartialBlock[]> {
+        await this.assertConsistency()
+
+        let requests = fromBlockHeight == null
+            ? this.requests
+            : applyRangeBound(this.requests, {from: fromBlockHeight})
+
+        if (requests.length == 0) return
+
+        if (this.gatewaySettings) {
+            let agent = new HttpAgent({keepAlive: true})
+            try {
+                let archive = this.createGateway(agent)
+                let height = await archive.getFinalizedHeight()
+                let from = requests[0].range.from
+                if (height > from || !this.graphql) {
+                    for await (let batch of archive.getBlockStream(requests, !!this.graphql)) {
+                        yield batch
+                        from = last(batch).header.height + 1
+                    }
+                    requests = applyRangeBound(requests, {from})
+                }
+            } finally {
+                agent.close()
+            }
+        }
+
+        if (requests.length == 0) return
+
+        assert(this.graphql)
+
+        yield* this.graphql.getBlockStream(requests)
+    }
+
+    private createGateway(agent?: HttpAgent): FuelGateway {
+        assert(this.gatewaySettings)
+
+        let http = new HttpClient({
+            headers: {
+                'x-squid-id': this.getSquidId()
+            },
+            agent
+        })
+
+        return new FuelGateway(
+            new ArchiveClient({
+                http,
+                url: this.gatewaySettings.url,
+                queryTimeout: this.gatewaySettings.requestTimeout,
+            })
+        )
+    }
+
+    private createGraphqlDataSource(settings: GraphqlSettings): GraphqlDataSource {
+        let dataSource = new HttpDataSource({
+            client: new HttpClient({baseUrl: settings.url}),
+            strideConcurrency: settings.strideConcurrency,
+            strideSize: settings.strideSize,
+        })
+        return new GraphqlDataSource(dataSource)
+    }
+
+    @def
+    private getSquidId(): string {
+        return getOrGenerateSquidId()
+    }
+}