@subsquid/solana-stream 1.0.0-portal-api.d5861e → 1.0.0

package/src/builder.ts

@@ -0,0 +1,205 @@
+import {PortalClient, PortalClientOptions} from '@subsquid/portal-client'
+import {getOrGenerateSquidId} from '@subsquid/util-internal-processor-tools'
+import {applyRangeBound, mergeRangeRequests, Range, RangeRequest, RangeRequestList} from '@subsquid/util-internal-range'
+import assert from 'assert'
+import {DEFAULT_FIELDS, FieldSelection} from './data/model'
+import {
+    BalanceRequest,
+    DataRequest,
+    InstructionRequest,
+    LogRequest,
+    RewardRequest,
+    TokenBalanceRequest,
+    TransactionRequest,
+} from './data/request'
+import {PortalSolanaDataSource} from './portal/source'
+import {type Query, QueryBuilder} from './query'
+import type {SolanaDataSource} from './source'
+
+interface BlockRange {
+    range?: Range
+}
+
+export class DataSourceBuilder<F extends FieldSelection = typeof DEFAULT_FIELDS> {
+    private requests: RangeRequest<DataRequest>[] = []
+    private fields?: FieldSelection
+    private blockRange?: Range
+    private portal?: PortalClient | PortalClientOptions
+
+    /**
+     * Set SQD Network Portal endpoint.
+     *
+     * SQD Network allows to get data from blocks up to
+     * infinite times faster and more efficient than via regular RPC.
+     *
+     * @example
+     * source.setPortal('https://portal.sqd.dev/datasets/solana-mainnet')
+     */
+    setPortal(portal: string | PortalClientOptions | PortalClient): this {
+        if (typeof portal == 'string') {
+            this.portal = {url: portal}
+        } else {
+            this.portal = portal
+        }
+        return this
+    }
+
+    /**
+     * Limits the range of blocks to fetch.
+     *
+     * Note, that block heights should be used instead of slots.
+     */
+    setBlockRange(range?: Range): this {
+        this.blockRange = range
+        return this
+    }
+
+    /**
+     * Configure a set of fetched fields
+     */
+    setFields<F extends FieldSelection>(fields: F): DataSourceBuilder<F> {
+        this.fields = fields
+        return this as any
+    }
+
+    /**
+     * Add a portal query - a set of item filters that share a block range.
+     *
+     * Accepts either a prebuilt {@link Query} object or a
+     * {@link QueryBuilder}, in which case {@link QueryBuilder#build} is
+     * invoked automatically.
+     *
+     * @example
+     * ```ts
+     * source.addQuery(new QueryBuilder()
+     *     .setRange({from: 250_000_000})
+     *     .addInstruction({where: {programId: [PROGRAM_ID]}, include: {logs: true}})
+     *     .addTransaction({where: {feePayer: [FEE_PAYER]}})
+     * )
+     * ```
+     */
+    addQuery(query: Query | QueryBuilder): this {
+        this.requests.push(query instanceof QueryBuilder ? query.build() : query)
+        return this
+    }
+
+    /**
+     * Shorthand for {@link DataSourceBuilder#addQuery} with a query that
+     * only sets {@link QueryBuilder#includeAllBlocks} (and optionally a
+     * range).
+     */
+    includeAllBlocks(range?: Range): this {
+        return this.addQuery({
+            range: range ?? {from: 0},
+            request: {includeAllBlocks: true},
+        })
+    }
+
+    /**
+     * Shorthand for {@link DataSourceBuilder#addQuery} with a query that
+     * contains a single transaction filter.
+     */
+    addTransaction(options: TransactionRequest & BlockRange): this {
+        let {range, ...req} = options
+        return this.addQuery({
+            range: range ?? {from: 0},
+            request: {transactions: [req]},
+        })
+    }
+
+    /**
+     * Shorthand for {@link DataSourceBuilder#addQuery} with a query that
+     * contains a single instruction filter.
+     */
+    addInstruction(options: InstructionRequest & BlockRange): this {
+        let {range, ...req} = options
+        return this.addQuery({
+            range: range ?? {from: 0},
+            request: {instructions: [req]},
+        })
+    }
+
+    /**
+     * Shorthand for {@link DataSourceBuilder#addQuery} with a query that
+     * contains a single log filter.
+     */
+    addLog(options: LogRequest & BlockRange): this {
+        let {range, ...req} = options
+        return this.addQuery({
+            range: range ?? {from: 0},
+            request: {logs: [req]},
+        })
+    }
+
+    /**
+     * Shorthand for {@link DataSourceBuilder#addQuery} with a query that
+     * contains a single balance filter.
+     */
+    addBalance(options: BalanceRequest & BlockRange): this {
+        let {range, ...req} = options
+        return this.addQuery({
+            range: range ?? {from: 0},
+            request: {balances: [req]},
+        })
+    }
+
+    /**
+     * Shorthand for {@link DataSourceBuilder#addQuery} with a query that
+     * contains a single token balance filter.
+     */
+    addTokenBalance(options: TokenBalanceRequest & BlockRange): this {
+        let {range, ...req} = options
+        return this.addQuery({
+            range: range ?? {from: 0},
+            request: {tokenBalances: [req]},
+        })
+    }
+
+    /**
+     * Shorthand for {@link DataSourceBuilder#addQuery} with a query that
+     * contains a single reward filter.
+     */
+    addReward(options: RewardRequest & BlockRange): this {
+        let {range, ...req} = options
+        return this.addQuery({
+            range: range ?? {from: 0},
+            request: {rewards: [req]},
+        })
+    }
+
+    private getRequests(): RangeRequestList<DataRequest> {
+        function concat<T>(a?: T[], b?: T[]): T[] | undefined {
+            let result: T[] = []
+            if (a) {
+                result.push(...a)
+            }
+            if (b) {
+                result.push(...b)
+            }
+            return result.length == 0 ? undefined : result
+        }
+
+        let requests = mergeRangeRequests(this.requests, (a, b) => {
+            return {
+                includeAllBlocks: a.includeAllBlocks || b.includeAllBlocks,
+                transactions: concat(a.transactions, b.transactions),
+                instructions: concat(a.instructions, b.instructions),
+                logs: concat(a.logs, b.logs),
+                balances: concat(a.balances, b.balances),
+                tokenBalances: concat(a.tokenBalances, b.tokenBalances),
+                rewards: concat(a.rewards, b.rewards),
+            }
+        })
+
+        return applyRangeBound(requests, this.blockRange)
+    }
+
+    build(): SolanaDataSource<F> {
+        assert(this.portal, 'Portal settings not set')
+
+        let portal = this.portal instanceof PortalClient ? this.portal : new PortalClient(this.portal)
+        return new PortalSolanaDataSource<F>(portal, this.fields ?? DEFAULT_FIELDS, () => this.getRequests(), {
+            squidId: getOrGenerateSquidId(),
+        })
+    }
+}

package/src/data/fields.ts

@@ -1,26 +1,6 @@
-import {DEFAULT_FIELDS, FieldSelection} from './model'
 import {Selector} from './type-util'
 
 
-function merge<Keys extends string>(def: Selector<Keys>, requested: Selector<Keys> = {}): Selector<Keys> {
-    let fields: Selector<Keys> = {}
-
-    for (let key in def) {
-        if (requested[key] !== false) {
-            fields[key] = def[key]
-        }
-    }
-
-    for (let key in requested) {
-        if (requested[key]) {
-            fields[key] = true
-        }
-    }
-
-    return fields
-}
-
-
 export function project<T>(fields: Selector<keyof T> | undefined, obj: T): Partial<T> {
     if (fields == null) return {}
     let result: Partial<T> = {}

package/src/data/model.ts

@@ -10,16 +10,16 @@ import type {
 } from './partial'
 import type {GetFields, Select, Selector, Simplify} from './type-util'
 
-/**
- * Hex encoded binary string
- */
+/** Hex encoded binary string. */
 export type Bytes = string
 
-/**
- * Base58 encoded binary string
- */
+/** Base58 encoded binary string. */
 export type Base58Bytes = string
 
+/**
+ * User-facing field selection. Each section is an optional `Selector`
+ * over that section's non-required fields.
+ */
 export interface FieldSelection {
     block?: Selector<Exclude<keyof data.BlockHeader, BlockRequiredFields>>
     transaction?: Selector<Exclude<keyof data.Transaction, TransactionRequiredFields>>
@@ -30,6 +30,13 @@ export interface FieldSelection {
     reward?: Selector<Exclude<keyof data.Reward, RewardRequiredFields>>
 }
 
+export type FieldKey = keyof FieldSelection
+
+/**
+ * Default field set applied when {@link DataSourceBuilder#setFields} is not
+ * called.  `as const` preserves the literal `true` values so they feed through
+ * {@link GetFields} correctly.
+ */
 export const DEFAULT_FIELDS = {
     block: {
         timestamp: true,
@@ -67,38 +74,27 @@ export const DEFAULT_FIELDS = {
         lamports: true,
         rewardType: true,
     },
-} as const
+} as const satisfies FieldSelection
 
-type Item<Data, RequiredFields extends keyof Data, F extends FieldSelection, K extends keyof FieldSelection> = Simplify<
-    Pick<Data, RequiredFields> & Select<Data, GetFields<FieldSelection, typeof DEFAULT_FIELDS, F, K>>
+/**
+ * A concrete item type derived from a raw data shape, the always-present
+ * required keys, and a per-section field selection.
+ */
+type Item<Data, Required extends keyof Data, F extends FieldSelection, K extends FieldKey> = Simplify<
+    Pick<Data, Required> & Select<Data, GetFields<F, K>>
 >
 
 export type BlockHeader<F extends FieldSelection = {}> = Item<data.BlockHeader, BlockRequiredFields, F, 'block'>
 
-export type Transaction<F extends FieldSelection = {}> = Item<
-    data.Transaction,
-    TransactionRequiredFields,
-    F,
-    'transaction'
->
+export type Transaction<F extends FieldSelection = {}> = Item<data.Transaction, TransactionRequiredFields, F, 'transaction'>
 
-export type Instruction<F extends FieldSelection = {}> = Item<
-    data.Instruction,
-    InstructionRequiredFields,
-    F,
-    'instruction'
->
+export type Instruction<F extends FieldSelection = {}> = Item<data.Instruction, InstructionRequiredFields, F, 'instruction'>
 
 export type LogMessage<F extends FieldSelection = {}> = Item<data.LogMessage, LogRequiredFields, F, 'log'>
 
 export type Balance<F extends FieldSelection = {}> = Item<data.Balance, BalanceRequiredFields, F, 'balance'>
 
-export type TokenBalance<F extends FieldSelection = {}> = Item<
-    data.TokenBalance,
-    TokenBalanceRequiredFields,
-    F,
-    'tokenBalance'
->
+export type TokenBalance<F extends FieldSelection = {}> = Item<data.TokenBalance, TokenBalanceRequiredFields, F, 'tokenBalance'>
 
 export type Reward<F extends FieldSelection = {}> = Item<data.Reward, RewardRequiredFields, F, 'reward'>
 

package/src/data/request.ts

@@ -1,9 +1,8 @@
 import {LogMessage} from '@subsquid/solana-normalization'
-import {Base58Bytes, FieldSelection} from './model'
+import {Base58Bytes} from './model'
 
 
-export interface DataRequest<F extends FieldSelection> {
-    fields?: F
+export interface DataRequest {
     includeAllBlocks?: boolean
     transactions?: TransactionRequest[]
     instructions?: InstructionRequest[]

package/src/data/type-util.ts

@@ -1,42 +1,44 @@
-export type Simplify<T> = {
-    [K in keyof T]: T[K]
-} & {}
-
-
-export type ExcludeUndefined<T> = {
-    [K in keyof T as undefined extends T[K] ? never : K]: T[K]
-} & {}
-
-
-export type GetFields<
-    FieldSelectionType,
-    Defaults extends FieldSelectionType,
-    Selection extends FieldSelectionType,
-    K extends keyof FieldSelectionType
-> = TrueFields<MergeDefault<Selection[K], Defaults[K]>>
-
-
-type MergeDefault<T, D> = Simplify<
-    undefined extends T ? D : Omit<D, keyof ExcludeUndefined<T>> & ExcludeUndefined<T>
->
-
-
-type TrueFields<F> = keyof {
-    [K in keyof F as true extends F[K] ? K : never]: true
-}
-
-
+/**
+ * Flattens an intersection type into a single object shape for nicer IDE hover
+ * output. The trailing `& {}` is a no-op at the type level but forces the
+ * compiler to normalize the representation.
+ */
+export type Simplify<T> = {[K in keyof T]: T[K]} & {}
+
+/**
+ * Extracts the string keys of `F` whose values include the literal `true`.
+ *
+ * @example
+ * type Keys = TrueFields<{a: true; b: false; c?: boolean}>
+ * // "a" | "c"  -- `c`'s type includes `true` through `boolean`
+ */
+export type TrueFields<F> = {
+    [K in keyof F]-?: true extends F[K] ? K : never
+}[keyof F]
+
+/**
+ * Picks the set of truthy field keys for a given section `K` of a
+ * `FieldSelection`-shaped type `F`.
+ */
+export type GetFields<F, K extends keyof F> = TrueFields<F[K]>
+
+/**
+ * Picks `Fields` from `T` while distributing over unions, so members of a
+ * discriminated union are narrowed independently.
+ */
 export type Select<T, Fields> = T extends any ? Simplify<Pick<T, Extract<keyof T, Fields>>> : never
 
-
-export type Selector<Fields extends string | number | symbol> = {
+/**
+ * Optional boolean flag map over a finite set of string keys.
+ *
+ * Shape: `{[K in Fields]?: boolean}` - equivalent to `Partial<Record<Fields, boolean>>`.
+ */
+export type Selector<Fields extends PropertyKey> = {
     [P in Fields]?: boolean
 }
 
-
-export type MakePartial<T, Required extends keyof T> = Simplify<
-    Pick<T, Required> &
-    {
-        [K in keyof T as K extends Required ? never : K]+?: T[K]
-    }
->
+/**
+ * Produces a type with `Required` keys kept as-is and all other keys made
+ * optional.
+ */
+export type MakePartial<T, Required extends keyof T> = Simplify<Pick<T, Required> & Partial<Omit<T, Required>>>

package/src/index.ts

@@ -1,3 +1,6 @@
 export * from './data/model'
 export * from './instruction'
+export * from './builder'
+export * from './query'
 export * from './source'
+export * from './portal/source'

package/src/{merge.test.ts → portal/merge.test.ts}

@@ -1,4 +1,4 @@
-import {describe, it} from 'node:test'
+import { describe, it } from 'vitest'
 import assert from 'assert'
 import {
     mergeItems,
@@ -8,7 +8,7 @@ import {
     BALANCE_FILTER_KEYS,
     TOKEN_BALANCE_FILTER_KEYS,
     REWARD_FILTER_KEYS,
-} from './portal/merge'
+} from './merge'
 
 describe('mergeItems', function () {
     describe('instructions', function () {