@subsquid/portal-client 0.0.0

package/lib/query/substrate.d.ts

@@ -0,0 +1,180 @@
+import type { Select, Selector, Trues, Hex, Simplify, PortalQuery } from './common';
+/**
+ * @example 'Balances.Transfer'
+ */
+export type QualifiedName = string & {};
+export type BlockHeaderFields = {
+    /**
+     * Block height
+     */
+    number: number;
+    /**
+     * Block hash
+     */
+    hash: Hex;
+    /**
+     * Hash of the parent block
+     */
+    parentHash: Hex;
+    /**
+     * Root hash of the state merkle tree
+     */
+    stateRoot: Hex;
+    /**
+     * Root hash of the extrinsics merkle tree
+     */
+    extrinsicsRoot: Hex;
+    digest: {
+        logs: Hex[];
+    };
+    specName: string;
+    specVersion: number;
+    implName: string;
+    implVersion: number;
+    /**
+     * Block timestamp as set by `timestamp.now()` (unix epoch ms, compatible with `Date`).
+     */
+    timestamp?: number;
+    /**
+     * Account address of block validator
+     */
+    validator?: Hex;
+};
+export type ExtrinsicSignatureFields = {
+    address: unknown;
+    signature: unknown;
+    signedExtensions: unknown;
+};
+export type ExtrinsicFields = {
+    /**
+     * Ordinal index in the extrinsics array of the current block
+     */
+    index: number;
+    version: number;
+    signature?: ExtrinsicSignatureFields;
+    fee?: bigint;
+    tip?: bigint;
+    error?: unknown;
+    success?: boolean;
+    /**
+     * Blake2b 128-bit hash of the raw extrinsic
+     */
+    hash?: Hex;
+};
+export type CallFields = {
+    extrinsicIndex: number;
+    address: number[];
+    name: QualifiedName;
+    args: unknown;
+    origin?: unknown;
+    /**
+     * Call error.
+     *
+     * Absence of error doesn't imply that the call was executed successfully,
+     * check {@link success} property for that.
+     */
+    error?: unknown;
+    success?: boolean;
+    _ethereumTransactTo?: Hex;
+    _ethereumTransactSighash?: Hex;
+};
+export type EventFields = {
+    /**
+     * Ordinal index in the event array of the current block
+     */
+    index: number;
+    /**
+     * Event name
+     */
+    name: QualifiedName;
+    args: unknown;
+    phase: 'Initialization' | 'ApplyExtrinsic' | 'Finalization';
+    extrinsicIndex?: number;
+    callAddress?: number[];
+    /**
+     * This field is not supported by all currently deployed archives.
+     * Requesting it may cause internal error.
+     */
+    topics: Hex[];
+    _evmLogAddress?: Hex;
+    _evmLogTopics?: Hex[];
+    _contractAddress?: Hex;
+    _gearProgramId?: Hex;
+};
+export type BlockHeaderFieldSelection = Simplify<Selector<keyof BlockHeaderFields> & {
+    number: true;
+    hash: true;
+}>;
+export type BlockHeader<T extends BlockHeaderFieldSelection = Trues<BlockHeaderFieldSelection>> = Select<BlockHeaderFields, T>;
+export type ExtrinsicFieldSelection = Selector<keyof ExtrinsicFields>;
+export type Extrinsic<T extends ExtrinsicFieldSelection = Trues<ExtrinsicFieldSelection>> = Select<ExtrinsicFields, T>;
+export type CallFieldSelection = Selector<keyof CallFields>;
+export type Call<T extends CallFieldSelection = Trues<CallFieldSelection>> = Select<CallFields, T>;
+export type EventFieldSelection = Selector<keyof EventFields>;
+export type Event<T extends EventFieldSelection = Trues<CallFieldSelection>> = Select<EventFields, T>;
+export type FieldSelection = {
+    block?: BlockHeaderFieldSelection;
+    extrinsic?: ExtrinsicFieldSelection;
+    call?: CallFieldSelection;
+    event?: EventFieldSelection;
+};
+export type EventRelations = {
+    extrinsic?: boolean;
+    call?: boolean;
+    stack?: boolean;
+};
+export type EventRequest = Simplify<{
+    name?: QualifiedName[];
+} & EventRelations>;
+export type CallRelations = {
+    extrinsic?: boolean;
+    stack?: boolean;
+    events?: boolean;
+};
+export type CallRequest = Simplify<{
+    name?: QualifiedName[];
+} & CallRelations>;
+export type EvmLogRequest = Simplify<{
+    address?: Hex[];
+} & EventRelations>;
+export type EthereumLogRequest = Simplify<{
+    address?: Hex[];
+    topic0?: Hex[];
+    topic1?: Hex[];
+    topic2?: Hex[];
+    topic3?: Hex[];
+} & EventRelations>;
+export type EthereumTransactRequest = Simplify<{
+    to?: Hex[];
+    sighash?: Hex[];
+} & CallRelations>;
+export type ContractsContractEmittedRequest = Simplify<{
+    address?: Hex[];
+} & EventRelations>;
+export type GearMessageQueuedRequest = Simplify<{
+    programId?: Hex[];
+} & EventRelations>;
+export type GearUserMessageSentRequest = Simplify<{
+    programId?: Hex[];
+} & EventRelations>;
+export type DataRequest = {
+    includeAllBlocks?: boolean;
+    events?: EventRequest[];
+    calls?: CallRequest[];
+    evmLogs?: EvmLogRequest[];
+    ethereumTransactions?: EthereumTransactRequest[];
+    contractsEvents?: ContractsContractEmittedRequest[];
+    gearMessagesQueued?: GearMessageQueuedRequest[];
+    gearUserMessagesSent?: GearUserMessageSentRequest[];
+};
+export type Query = Simplify<PortalQuery & {
+    type: 'substrate';
+    fields: FieldSelection;
+} & DataRequest>;
+export type Block<F extends FieldSelection> = Simplify<{
+    header: BlockHeader<F['block'] & {}>;
+    events?: Event<F['event'] & {}>[];
+    calls?: Call<F['call'] & {}>[];
+    extrinsics?: Extrinsic<F['extrinsic'] & {}>[];
+}>;
+//# sourceMappingURL=substrate.d.ts.map

package/lib/query/substrate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"substrate.d.ts","sourceRoot":"","sources":["../../src/query/substrate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAe,WAAW,EAAC,MAAM,UAAU,CAAA;AAE9F;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,EAAE,CAAA;AAEvC,MAAM,MAAM,iBAAiB,GAAG;IAC5B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,IAAI,EAAE,GAAG,CAAA;IACT;;OAEG;IACH,UAAU,EAAE,GAAG,CAAA;IACf;;OAEG;IACH,SAAS,EAAE,GAAG,CAAA;IACd;;OAEG;IACH,cAAc,EAAE,GAAG,CAAA;IACnB,MAAM,EAAE;QAAC,IAAI,EAAE,GAAG,EAAE,CAAA;KAAC,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,SAAS,CAAC,EAAE,GAAG,CAAA;CAClB,CAAA;AAED,MAAM,MAAM,wBAAwB,GAAG;IACnC,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,OAAO,CAAA;IAClB,gBAAgB,EAAE,OAAO,CAAA;CAC5B,CAAA;AAED,MAAM,MAAM,eAAe,GAAG;IAC1B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,wBAAwB,CAAA;IACpC,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB;;OAEG;IACH,IAAI,CAAC,EAAE,GAAG,CAAA;CACb,CAAA;AAED,MAAM,MAAM,UAAU,GAAG;IACrB,cAAc,EAAE,MAAM,CAAA;IACtB,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,IAAI,EAAE,aAAa,CAAA;IACnB,IAAI,EAAE,OAAO,CAAA;IACb,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,mBAAmB,CAAC,EAAE,GAAG,CAAA;IACzB,wBAAwB,CAAC,EAAE,GAAG,CAAA;CACjC,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACtB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,IAAI,EAAE,aAAa,CAAA;IACnB,IAAI,EAAE,OAAO,CAAA;IACb,KAAK,EAAE,gBAAgB,GAAG,gBAAgB,GAAG,cAAc,CAAA;IAC3D,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB;;;OAGG;IACH,MAAM,EAAE,GAAG,EAAE,CAAA;IACb,cAAc,CAAC,EAAE,GAAG,CAAA;IACpB,aAAa,CAAC,EAAE,GAAG,EAAE,CAAA;IACrB,gBAAgB,CAAC,EAAE,GAAG,CAAA;IACtB,cAAc,CAAC,EAAE,GAAG,CAAA;CACvB,CAAA;AAED,MAAM,MAAM,yBAAyB,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,iBAAiB,CAAC,GAAG;IAAC,MAAM,EAAE,IAAI,CAAC;IAAC,IAAI,EAAE,IAAI,CAAA;CAAC,CAAC,CAAA;AAChH,MAAM,MAAM,WAAW,CAAC,CAAC,SAAS,yBAAyB,GAAG,KAAK,CAAC,yBAAyB,CAAC,IAAI,MAAM,CACpG,iBAAiB,EACjB,CAAC,CACJ,CAAA;AAED,MAAM,MAAM,uBAAuB,GAAG,QAAQ,CAAC,MAAM,eAAe,CAAC,CAAA;AACrE,MAAM,MAAM,SAAS,CAAC,CAAC,SAAS,uBAAuB,GAAG,KAAK,CAAC,uBAAuB,CAAC,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,CAAA;AAEtH,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CAAC,MAAM,UAAU,CAAC,CAAA;AAC3D,MAAM,MAAM,IAAI,CAAC,CAAC,SAAS,kBAAkB,GAAG,KAAK,CAAC,kBAAkB,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAA;AAElG,MAAM,MAAM,mBAAmB,GAAG,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAA;AAC7D,MAAM,MAAM,KAAK,CAAC,CAAC,SAAS,mBAAmB,GAAG,KAAK,CAAC,kBAAkB,CAAC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;AAErG,MAAM,MAAM,cAAc,GAAG;IACzB,KAAK,CAAC,EAAE,yBAAyB,CAAA;IACjC,SAAS,CAAC,EAAE,uBAAuB,CAAA;IACnC,IAAI,CAAC,EAAE,kBAAkB,CAAA;IACzB,KAAK,CAAC,EAAE,mBAAmB,CAAA;CAC9B,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IACzB,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,KAAK,CAAC,EAAE,OAAO,CAAA;CAClB,CAAA;AAED,MAAM,MAAM,YAAY,GAAG,QAAQ,CAC/B;IACI,IAAI,CAAC,EAAE,aAAa,EAAE,CAAA;CACzB,GAAG,cAAc,CACrB,CAAA;AAED,MAAM,MAAM,aAAa,GAAG;IACxB,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,WAAW,GAAG,QAAQ,CAAC;IAAC,IAAI,CAAC,EAAE,aAAa,EAAE,CAAA;CAAC,GAAG,aAAa,CAAC,CAAA;AAE5E,MAAM,MAAM,aAAa,GAAG,QAAQ,CAAC;IAAC,OAAO,CAAC,EAAE,GAAG,EAAE,CAAA;CAAC,GAAG,cAAc,CAAC,CAAA;AAExE,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CACrC;IACI,OAAO,CAAC,EAAE,GAAG,EAAE,CAAA;IACf,MAAM,CAAC,EAAE,GAAG,EAAE,CAAA;IACd,MAAM,CAAC,EAAE,GAAG,EAAE,CAAA;IACd,MAAM,CAAC,EAAE,GAAG,EAAE,CAAA;IACd,MAAM,CAAC,EAAE,GAAG,EAAE,CAAA;CACjB,GAAG,cAAc,CACrB,CAAA;AAED,MAAM,MAAM,uBAAuB,GAAG,QAAQ,CAAC;IAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC;IAAC,OAAO,CAAC,EAAE,GAAG,EAAE,CAAA;CAAC,GAAG,aAAa,CAAC,CAAA;AAE7F,MAAM,MAAM,+BAA+B,GAAG,QAAQ,CAAC;IAAC,OAAO,CAAC,EAAE,GAAG,EAAE,CAAA;CAAC,GAAG,cAAc,CAAC,CAAA;AAE1F,MAAM,MAAM,wBAAwB,GAAG,QAAQ,CAAC;IAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAA;CAAC,GAAG,cAAc,CAAC,CAAA;AAErF,MAAM,MAAM,0BAA0B,GAAG,QAAQ,CAAC;IAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAA;CAAC,GAAG,cAAc,CAAC,CAAA;AAEvF,MAAM,MAAM,WAAW,GAAG;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAC1B,MAAM,CAAC,EAAE,YAAY,EAAE,CAAA;IACvB,KAAK,CAAC,EAAE,WAAW,EAAE,CAAA;IACrB,OAAO,CAAC,EAAE,aAAa,EAAE,CAAA;IACzB,oBAAoB,CAAC,EAAE,uBAAuB,EAAE,CAAA;IAChD,eAAe,CAAC,EAAE,+BAA+B,EAAE,CAAA;IACnD,kBAAkB,CAAC,EAAE,wBAAwB,EAAE,CAAA;IAC/C,oBAAoB,CAAC,EAAE,0BAA0B,EAAE,CAAA;CACtD,CAAA;AAED,MAAM,MAAM,KAAK,GAAG,QAAQ,CACxB,WAAW,GAAG;IACV,IAAI,EAAE,WAAW,CAAA;IACjB,MAAM,EAAE,cAAc,CAAA;CACzB,GAAG,WAAW,CAClB,CAAA;AAED,MAAM,MAAM,KAAK,CAAC,CAAC,SAAS,cAAc,IAAI,QAAQ,CAAC;IACnD,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IACpC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAA;IACjC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,CAAA;IAC9B,UAAU,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,EAAE,CAAA;CAChD,CAAC,CAAA"}

package/lib/query/substrate.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=substrate.js.map

package/lib/query/substrate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"substrate.js","sourceRoot":"","sources":["../../src/query/substrate.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@subsquid/portal-client",
+  "version": "0.0.0",
+  "description": "SQD Portal API",
+  "license": "GPL-3.0-or-later",
+  "repository": "git@github.com:subsquid/squid.git",
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "lib/client.js",
+  "files": [
+    "lib",
+    "src",
+    "!src/*.test.*",
+    "!lib/*.test.*"
+  ],
+  "dependencies": {
+    "@subsquid/util-internal": "^3.2.0"
+  },
+  "peerDependencies": {
+    "@subsquid/http-client": "^1.6.1"
+  },
+  "peerDependenciesMeta": {},
+  "devDependencies": {
+    "@subsquid/http-client": "^1.6.1",
+    "@types/node": "^18.18.14",
+    "typescript": "5.5.4"
+  },
+  "scripts": {
+    "build": "rm -rf lib && tsc"
+  }
+}

package/src/client.example.ts

@@ -0,0 +1,167 @@
+import {BlockRef, createQuery, isForkException, PortalClient} from './client'
+
+const portalUrls = {
+    evm: 'https://portal.sqd.dev/datasets/ethereum-mainnet',
+    solana: 'https://portal.sqd.dev/datasets/solana-mainnet',
+}
+
+const queries = {
+    evm: createQuery({
+        type: 'evm',
+        fromBlock: 23_000_000,
+        fields: {
+            block: {
+                number: true,
+                hash: true,
+                timestamp: true,
+            },
+            transaction: {
+                from: true,
+                to: true,
+                hash: true,
+            },
+            log: {
+                address: true,
+                topics: true,
+                data: true,
+                transactionHash: true,
+                logIndex: true,
+                transactionIndex: true,
+            },
+            stateDiff: {
+                kind: true,
+                next: true,
+                prev: true,
+            },
+        },
+        logs: [
+            {
+                address: ['0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48'],
+                topic0: ['0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'],
+            },
+        ],
+    }),
+    solana: createQuery({
+        type: 'solana',
+        fromBlock: 358_600_000,
+        fields: {
+            block: {number: true, timestamp: true, hash: true, parentHash: true},
+            transaction: {signatures: true, err: true, transactionIndex: true},
+            instruction: {
+                programId: true,
+                accounts: true,
+                data: true,
+                isCommitted: true,
+                transactionIndex: true,
+                instructionAddress: true,
+            },
+        },
+        instructions: [
+            {
+                programId: ['whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc'],
+                d8: ['0xf8c69e91e17587c8'],
+                isCommitted: true,
+                innerInstructions: true,
+            },
+        ],
+    }),
+}
+
+async function main() {
+    const queryType = 'evm'
+    const query = queries[queryType]
+
+    let portal = new PortalClient({
+        url: portalUrls[queryType],
+        http: {
+            retryAttempts: Infinity,
+        },
+        minBytes: 50 * 1024 * 1024,
+    })
+
+    let coldHead: BlockRef | undefined = undefined
+    let hotHeads: BlockRef[] = []
+    while (true) {
+        const currentQuery = {...query}
+
+        let head = hotHeads[hotHeads.length - 1] ?? coldHead
+        if (head != null && head.number > currentQuery.fromBlock) {
+            currentQuery.fromBlock = head.number + 1
+            currentQuery.parentBlockHash = head.hash
+        }
+
+        try {
+            for await (let {blocks, finalizedHead} of portal.getStream(query)) {
+                if (head && blocks.length > 0 && head.number >= blocks[0].header.number) {
+                    throw new Error('Data is not continuous')
+                }
+
+                let unfinalizedIndex = 0
+                if (finalizedHead) {
+                    unfinalizedIndex = blocks.findIndex((b) => b.header.number > finalizedHead?.number)
+                }
+
+                // all new blocks are finalized
+                if (unfinalizedIndex < 0) {
+                    const finalizedRef = blocks[blocks.length - 1].header
+                    coldHead = {number: finalizedRef.number, hash: finalizedRef.hash}
+                    // finalize all hot heads
+                    hotHeads = []
+                } else {
+                    const finalizedRef = finalizedHead ?? blocks[unfinalizedIndex - 1]?.header
+                    coldHead = finalizedRef ?? coldHead
+
+                    // finalize all hot heads that are older than the cold head
+                    if (coldHead) {
+                        let finalizeIndex = hotHeads.findIndex((h) => h.number > coldHead!.number)
+                        hotHeads = finalizeIndex < 0 ? [] : hotHeads.slice(finalizeIndex)
+                    }
+
+                    // process unfinalized blocks
+                    for (let i = unfinalizedIndex; i < blocks.length; i++) {
+                        hotHeads.push({number: blocks[i].header.number, hash: blocks[i].header.hash})
+                    }
+                }
+
+                head = hotHeads[hotHeads.length - 1] ?? coldHead
+                let portalHead = Math.max(head.number, finalizedHead?.number ?? -1)
+                console.log(`progress: ${head.number} / ${portalHead}` + `, blocks: ${blocks.length}`)
+                console.log(`  \u001b[2mcold head: ${coldHead ? formatRef(coldHead) : 'N/A'}\u001b[0m`)
+                console.log(`  \u001b[2mhot heads: ${hotHeads.map((h) => formatRef(h)).join(', ') || 'N/A'}\u001b[0m`)
+            }
+            break
+        } catch (e) {
+            if (!isForkException(e)) throw e
+
+            let chain = coldHead ? [coldHead, ...hotHeads] : hotHeads
+            let rollbackIndex = findRollbackIndex(chain, e.lastBlocks)
+            if (rollbackIndex === -1) throw new Error('Unable to process fork')
+
+            const rollbackHead = chain[rollbackIndex]
+            console.warn(`detected fork at block ${rollbackHead.number} (${e.head.number - rollbackHead.number} depth)`)
+
+            hotHeads = chain.slice(1, rollbackIndex + 1)
+            head = hotHeads[hotHeads.length - 1] ?? coldHead
+        }
+    }
+}
+
+function findRollbackIndex(chainA: BlockRef[], chainB: BlockRef[]) {
+    let i = 0
+    let j = 0
+    for (; i < chainA.length; i++) {
+        const blockA = chainA[i]
+        for (; j < chainB.length; j++) {
+            let blockB = chainB[j]
+            if (blockB.number > blockA.number) break
+            if (blockB.number === blockA.number && blockB.hash !== blockA.hash) return i - 1
+        }
+    }
+    return i - 1
+}
+
+function formatRef(ref: BlockRef) {
+    return `${ref.number}#${ref.hash.slice(2, 8)}`
+}
+
+main()