@subsquid/solana-stream 1.0.0-portal-api.d5861e → 1.0.0

package/src/portal/source.ts

@@ -1,130 +1,146 @@
 import {isForkException as isPortalForkException, PortalClient, solana} from '@subsquid/portal-client'
 import {maybeLast} from '@subsquid/util-internal'
 import {
+    BlockBatch,
     BlockRef,
-    DataSource,
-    DataSourceStreamOptions,
     ForkException,
-    type BlockBatch,
-    TemplateRegistry,
+    StreamRequest,
 } from '@subsquid/util-internal-data-source'
-import {applyRangeBound, FiniteRange, getSize, RangeRequestList, type RangeRequest} from '@subsquid/util-internal-range'
 import {
-    Block,
-    FieldSelection,
-    type Balance,
-    type BlockHeader,
-    type Instruction,
-    type LogMessage,
-    type Reward,
-    type TokenBalance,
-    type Transaction,
-} from '../data/model'
+    applyRangeBound,
+    FiniteRange,
+    getSize,
+    RangeRequest,
+    RangeRequestList,
+} from '@subsquid/util-internal-range'
+import assert from 'assert'
+import {Block, FieldSelection} from '../data/model'
 import {DataRequest} from '../data/request'
 import {
-    mergeItems,
-    TX_FILTER_KEYS,
+    BALANCE_FILTER_KEYS,
     INSTRUCTION_FILTER_KEYS,
     LOG_FILTER_KEYS,
-    BALANCE_FILTER_KEYS,
-    TOKEN_BALANCE_FILTER_KEYS,
+    mergeItems,
     REWARD_FILTER_KEYS,
+    TOKEN_BALANCE_FILTER_KEYS,
+    TX_FILTER_KEYS,
 } from './merge'
-import assert from 'assert'
+import type {SolanaDataSource} from '../source'
 
-export type RangeRequestResolver<F extends FieldSelection> =
-    | RangeRequestList<DataRequest<F>>
-    | ((registry?: TemplateRegistry) => RangeRequestList<DataRequest<F>>)
+export type RangeRequestResolver =
+    | RangeRequestList<DataRequest>
+    | (() => RangeRequestList<DataRequest>)
 
-export class PortalDataSource<F extends FieldSelection> implements DataSource<Block<F>> {
+export class PortalSolanaDataSource<F extends FieldSelection> implements SolanaDataSource<F> {
     constructor(
         private client: PortalClient,
-        private requests: RangeRequestResolver<F>,
-        private opts?: {squidId?: string}
+        private fields: FieldSelection,
+        private requests: RangeRequestResolver,
+        private opts?: {squidId?: string},
     ) {}
 
-    getHead(): Promise<BlockRef | undefined> {
-        return this.client.getHead({headers: this.getHeaders()})
+    async getHead(): Promise<BlockRef> {
+        let head = await this.client.getHead({headers: this.getHeaders()})
+        assert(head, 'portal has no chain head')
+        return head
     }
 
-    getFinalizedHead(): Promise<BlockRef | undefined> {
-        return this.client.getFinalizedHead({headers: this.getHeaders()})
+    async getFinalizedHead(): Promise<BlockRef> {
+        let head = await this.client.getFinalizedHead({headers: this.getHeaders()})
+        assert(head, 'portal has no finalized head')
+        return head
     }
 
-    getFinalizedStream(opts?: DataSourceStreamOptions): AsyncIterable<BlockBatch<Block<F>>> {
+    getFinalizedStream(opts?: StreamRequest): AsyncIterable<BlockBatch<Block<F>>> {
         return this._getStream(opts, true)
     }
 
-    getStream(opts?: DataSourceStreamOptions): AsyncIterable<BlockBatch<Block<F>>> {
+    getStream(opts?: StreamRequest): AsyncIterable<BlockBatch<Block<F>>> {
         return this._getStream(opts, false)
     }
 
     getBlocksCountInRange(range: FiniteRange): number {
-        return getSize(this.resolveRequests().map(r => r.range), range)
+        return getSize(
+            this.resolveRequests().map((r) => r.range),
+            range,
+        )
     }
 
-    private resolveRequests(registry?: TemplateRegistry): RangeRequestList<DataRequest<F>> {
-        return typeof this.requests === 'function' ? this.requests(registry) : this.requests
+    private resolveRequests(): RangeRequestList<DataRequest> {
+        return typeof this.requests === 'function' ? this.requests() : this.requests
     }
 
-    private async *_getStream(
-        opts?: DataSourceStreamOptions,
-        finalized?: boolean
-    ): AsyncIterable<BlockBatch<Block<F>>> {
-        let requests = applyRangeBound(this.resolveRequests(opts?.templateRegistry), opts?.from != null ? {from: opts.from} : undefined)
+    private async *_getStream(opts?: StreamRequest, finalized?: boolean): AsyncIterable<BlockBatch<Block<F>>> {
+        let requests = applyRangeBound(this.resolveRequests(), opts?.from != null ? {from: opts.from} : undefined)
         if (requests.length === 0) return
 
         let streamOptions = {request: {headers: this.getHeaders()}}
         let parentHash = opts?.parentHash
-
-        for (let i = 0; i < requests.length; i++) {
-            let req = requests[i]
-            let query = mapRequest(req, parentHash)
-            let stream = this.client.getStream(query, streamOptions, finalized)
-
-            try {
-                for await (let {blocks, meta} of stream) {
-                    yield {
-                        blocks: blocks.map((block) => mapBlock(block)),
-                        finalizedHead: getHead(meta.finalizedHeadNumber, meta.finalizedHeadHash),
-                    }
-
-                    let lastBlock = maybeLast(blocks)?.header
-                    if (lastBlock != null) {
-                        parentHash = lastBlock.hash
-                    }
+        let nextBlock = opts?.from
+
+        // Stream a request from the portal and keep (parentHash, nextBlock)
+        // in sync with the last block of every batch.
+        let drain = async function* (
+            this: PortalSolanaDataSource<F>,
+            req: RangeRequest<DataRequest>,
+        ): AsyncIterable<BlockBatch<Block<F>>> {
+            for await (let {blocks, meta} of this.client.getStream(
+                mapRequest(req, this.fields, parentHash),
+                streamOptions,
+                finalized,
+            )) {
+                yield {
+                    blocks: blocks.map((block) => mapBlock(block)),
+                    finalizedHead: getHead(meta.finalizedHeadNumber, meta.finalizedHeadHash),
                 }
 
-                // finalize range
-                assert(req.range.to != null)
-
-                let nextReq = requests[i + 1]
-                let gapRange = {from: req.range.to + 1, to: nextReq ? nextReq.range.from - 1 : undefined}
-                if (gapRange.from === gapRange.to) continue
-
-                for await (let {blocks, meta} of this.client.getStream(
-                    mapRequest({range: gapRange, request: {fields: req.request.fields}}, parentHash),
-                    streamOptions,
-                    finalized
-                )) {
-                    let finalizedHead = getHead(meta.finalizedHeadNumber, meta.finalizedHeadHash)
-                    if (finalizedHead && req.range.to <= finalizedHead.number) {
-                        parentHash = undefined
-                        break
-                    }
-
-                    let lastBlock = maybeLast(blocks)?.header
-                    if (lastBlock != null) {
-                        parentHash = lastBlock.hash
-                    }
+                let lastBlock = maybeLast(blocks)?.header
+                if (lastBlock != null) {
+                    parentHash = lastBlock.hash
+                    nextBlock = lastBlock.number + 1
+                }
+            }
+        }.bind(this)
+
+        // Advance the stream until the given block becomes finalized, after
+        // which chain-continuity info is no longer needed and is dropped.
+        // When `to` is undefined the stream is open-ended and we wait on the
+        // current tail (nextBlock - 1).
+        let finalize = async (to?: number) => {
+            if (nextBlock == null) return
+            if (to != null && nextBlock > to) return
+            let target = to ?? nextBlock - 1
+            for await (let {finalizedHead} of drain({range: {from: nextBlock, to}, request: {}})) {
+                if (finalizedHead && target <= finalizedHead.number) {
+                    parentHash = undefined
+                    nextBlock = undefined
+                    return
                 }
-            } catch (e: unknown) {
-                if (isPortalForkException(e)) {
-                    throw new ForkException(e.blockNumber, e.parentBlockHash, e.previousBlocks)
+            }
+        }
+
+        try {
+            for (let req of requests) {
+                // Close any gap preceding this query so parentHash lands on
+                // req.range.from (or is safely dropped).
+                await finalize(req.range.from - 1)
+
+                if (nextBlock !== req.range.from) {
+                    parentHash = undefined
                 }
 
-                throw e
+                yield* drain(req)
             }
+
+            // After the last query, wait for its range to finalize before
+            // ending the stream.
+            await finalize()
+        } catch (e: unknown) {
+            if (isPortalForkException(e)) {
+                throw new ForkException(e.blockNumber, e.parentBlockHash, e.previousBlocks)
+            }
+
+            throw e
         }
     }
 
@@ -140,10 +156,11 @@ function getHead(number?: number | undefined, hash?: string | undefined): BlockR
     return {number, hash}
 }
 
-function mapRequest<F extends FieldSelection>(
-    req: RangeRequest<DataRequest<F>>,
-    parentBlockHash?: string
-): solana.Query<MapFieldSelection<F>> {
+function mapRequest(
+    req: RangeRequest<DataRequest>,
+    fields: FieldSelection,
+    parentBlockHash?: string,
+): solana.Query<MapFieldSelection> {
     let transactions = req.request.transactions?.map((tx) => ({...tx.where, ...tx.include}))
     let instructions = req.request.instructions?.map((ix) => ({...ix.where, ...ix.include}))
     let logs = req.request.logs?.map((log) => ({...log.where, ...log.include}))
@@ -155,7 +172,7 @@ function mapRequest<F extends FieldSelection>(
         fromBlock: req.range.from,
         toBlock: req.range.to === Infinity ? undefined : req.range.to,
         parentBlockHash: parentBlockHash,
-        fields: mapFieldSelection(req.request.fields),
+        fields: mapFieldSelection(fields),
         includeAllBlocks: req.request.includeAllBlocks,
         transactions: transactions && mergeItems(transactions, TX_FILTER_KEYS),
         instructions: instructions && mergeItems(instructions, INSTRUCTION_FILTER_KEYS),
@@ -166,21 +183,21 @@ function mapRequest<F extends FieldSelection>(
     }
 }
 
-function mapFieldSelection<F extends FieldSelection>(fields?: F) {
+function mapFieldSelection(fields: FieldSelection) {
     return {
-        block: fields?.block,
-        transaction: {...fields?.transaction, transactionIndex: true},
-        instruction: {...fields?.instruction, transactionIndex: true, instructionAddress: true},
-        log: {...fields?.log, logIndex: true, transactionIndex: true, instructionAddress: true},
-        balance: {...fields?.balance, transactionIndex: true, account: true},
-        tokenBalance: {...fields?.tokenBalance, transactionIndex: true, account: true},
-        reward: {...fields?.reward, pubkey: true},
+        block: fields.block,
+        transaction: {...fields.transaction, transactionIndex: true},
+        instruction: {...fields.instruction, transactionIndex: true, instructionAddress: true},
+        log: {...fields.log, logIndex: true, transactionIndex: true, instructionAddress: true},
+        balance: {...fields.balance, transactionIndex: true, account: true},
+        tokenBalance: {...fields.tokenBalance, transactionIndex: true, account: true},
+        reward: {...fields.reward, pubkey: true},
     } satisfies solana.FieldSelection
 }
 
-type MapFieldSelection<F extends FieldSelection> = ReturnType<typeof mapFieldSelection<F>>
+type MapFieldSelection = ReturnType<typeof mapFieldSelection>
 
-export function mapBlock<F extends FieldSelection>(rawBlock: solana.Block<MapFieldSelection<F>>): Block<F> {
+export function mapBlock<F extends FieldSelection>(rawBlock: solana.Block<MapFieldSelection>): Block<F> {
     let {number, hash, ...hdr} = rawBlock.header
     let header = {
         number,

package/src/query.ts

@@ -0,0 +1,112 @@
+import type {Range, RangeRequest} from '@subsquid/util-internal-range'
+import type {
+    BalanceRequest,
+    DataRequest,
+    InstructionRequest,
+    LogRequest,
+    RewardRequest,
+    TokenBalanceRequest,
+    TransactionRequest,
+} from './data/request'
+
+/**
+ * A single portal query - a set of item filters that share a block range.
+ *
+ * Produced by {@link QueryBuilder#build} and consumed by
+ * {@link DataSourceBuilder#addQuery}.
+ */
+export type Query = RangeRequest<DataRequest>
+
+/**
+ * Builder for a single portal query - a set of item filters that share a block range.
+ *
+ * Pass a {@link QueryBuilder} or the result of {@link QueryBuilder#build}
+ * to {@link DataSourceBuilder#addQuery} to register the query.
+ *
+ * @example
+ * ```ts
+ * let query = new QueryBuilder()
+ *     .setRange({from: 250_000_000})
+ *     .addInstruction({where: {programId: [PROGRAM_ID]}, include: {logs: true}})
+ *     .addTransaction({where: {feePayer: [FEE_PAYER]}})
+ *
+ * dataSource.addQuery(query)
+ * ```
+ */
+export class QueryBuilder {
+    private _range: Range = {from: 0}
+    private _request: DataRequest = {}
+
+    /**
+     * Restrict this query to the given block range.
+     *
+     * When omitted, the query applies from block 0 onwards.
+     */
+    setRange(range: Range): this {
+        this._range = range
+        return this
+    }
+
+    /**
+     * Fetch every block in the query's range, even blocks without any
+     * matching items.  Without this flag such blocks may be omitted.
+     */
+    includeAllBlocks(): this {
+        this._request.includeAllBlocks = true
+        return this
+    }
+
+    addTransaction(options: TransactionRequest): this {
+        ;(this._request.transactions ??= []).push(options)
+        return this
+    }
+
+    addInstruction(options: InstructionRequest): this {
+        ;(this._request.instructions ??= []).push(options)
+        return this
+    }
+
+    addLog(options: LogRequest): this {
+        ;(this._request.logs ??= []).push(options)
+        return this
+    }
+
+    addBalance(options: BalanceRequest): this {
+        ;(this._request.balances ??= []).push(options)
+        return this
+    }
+
+    addTokenBalance(options: TokenBalanceRequest): this {
+        ;(this._request.tokenBalances ??= []).push(options)
+        return this
+    }
+
+    addReward(options: RewardRequest): this {
+        ;(this._request.rewards ??= []).push(options)
+        return this
+    }
+
+    /**
+     * Produce an immutable {@link Query} describing the configured range
+     * and filters.  Subsequent mutations of this builder do not affect
+     * the returned object.
+     */
+    build(): Query {
+        return {
+            range: {...this._range},
+            request: cloneRequest(this._request),
+        }
+    }
+}
+
+function cloneRequest(request: DataRequest): DataRequest {
+    return {
+        includeAllBlocks: request.includeAllBlocks,
+        transactions: request.transactions?.slice(),
+        instructions: request.instructions?.slice(),
+        logs: request.logs?.slice(),
+        balances: request.balances?.slice(),
+        tokenBalances: request.tokenBalances?.slice(),
+        rewards: request.rewards?.slice(),
+    }
+}