@subsquid/portal-client 0.0.1-portal-api.4494d2

package/README.md

@@ -0,0 +1,3 @@
+# @subsquid/portal-client
+
+API for SQD Portal.

package/lib/client.d.ts

@@ -0,0 +1,35 @@
+import { HttpClient } from '@subsquid/http-client';
+import type { Logger } from '@subsquid/logger';
+export interface PortalQuery {
+    fromBlock: number;
+    toBlock?: number;
+}
+export interface Block {
+    header: {
+        number: number;
+        hash: string;
+    };
+}
+export interface Metadata {
+    isRealTime: boolean;
+}
+export interface PortalClientOptions {
+    url: string;
+    http?: HttpClient;
+    log?: Logger;
+    queryTimeout?: number;
+    bufferThreshold?: number;
+}
+export declare class PortalClient {
+    private url;
+    private http;
+    private queryTimeout;
+    private bufferThreshold;
+    constructor(options: PortalClientOptions);
+    private getDatasetUrl;
+    getHeight(): Promise<number>;
+    getMetadata(): Promise<Metadata>;
+    query<B extends Block = Block, Q extends PortalQuery = PortalQuery>(query: Q): Promise<B[]>;
+    stream<B extends Block = Block, Q extends PortalQuery = PortalQuery>(query: Q): AsyncIterable<B[]>;
+}
+//# sourceMappingURL=client.d.ts.map

package/lib/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAA;AAChD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAA;AAM5C,MAAM,WAAW,WAAW;IACxB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;CACnB;AAGD,MAAM,WAAW,KAAK;IAClB,MAAM,EAAE;QACJ,MAAM,EAAE,MAAM,CAAA;QACd,IAAI,EAAE,MAAM,CAAA;KACf,CAAA;CACJ;AAGD,MAAM,WAAW,QAAQ;IACrB,UAAU,EAAE,OAAO,CAAA;CACtB;AAGD,MAAM,WAAW,mBAAmB;IAChC,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,CAAC,EAAE,UAAU,CAAA;IACjB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,eAAe,CAAC,EAAE,MAAM,CAAA;CAC3B;AAGD,qBAAa,YAAY;IACrB,OAAO,CAAC,GAAG,CAAK;IAChB,OAAO,CAAC,IAAI,CAAY;IACxB,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,eAAe,CAAQ;gBAEnB,OAAO,EAAE,mBAAmB;IAOxC,OAAO,CAAC,aAAa;IAUf,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;IAU5B,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC;IAUtC,KAAK,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK,EAAE,CAAC,SAAS,WAAW,GAAG,WAAW,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAuBpF,MAAM,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK,EAAE,CAAC,SAAS,WAAW,GAAG,WAAW,EAAE,KAAK,EAAE,CAAC,GAAG,aAAa,CAAC,CAAC,EAAE,CAAC;CAuE5G"}

package/lib/client.js

@@ -0,0 +1,126 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PortalClient = void 0;
+const http_client_1 = require("@subsquid/http-client");
+const util_internal_1 = require("@subsquid/util-internal");
+const util_internal_archive_layout_1 = require("@subsquid/util-internal-archive-layout");
+const assert_1 = __importDefault(require("assert"));
+class PortalClient {
+    constructor(options) {
+        this.url = new URL(options.url);
+        this.http = options.http || new http_client_1.HttpClient({ log: options.log });
+        this.queryTimeout = options.queryTimeout ?? 180000;
+        this.bufferThreshold = options.bufferThreshold ?? 10 * 1024 * 1024;
+    }
+    getDatasetUrl(path) {
+        let u = new URL(this.url);
+        if (this.url.pathname.endsWith('/')) {
+            u.pathname += path;
+        }
+        else {
+            u.pathname += '/' + path;
+        }
+        return u.toString();
+    }
+    async getHeight() {
+        let res = await this.http.get(this.getDatasetUrl('height'), {
+            retryAttempts: 3,
+            httpTimeout: 10000,
+        });
+        let height = parseInt(res);
+        (0, assert_1.default)(Number.isSafeInteger(height));
+        return height;
+    }
+    async getMetadata() {
+        let res = await this.http.get(this.getDatasetUrl('metadata'), {
+            retryAttempts: 3,
+            httpTimeout: 10000,
+        });
+        return {
+            isRealTime: !!res.real_time
+        };
+    }
+    query(query) {
+        return this.http
+            .request('POST', this.getDatasetUrl(`stream`), {
+            json: query,
+            retryAttempts: 3,
+            httpTimeout: this.queryTimeout,
+        })
+            .catch((0, util_internal_1.withErrorContext)({
+            archiveQuery: query,
+        }))
+            .then((res) => {
+            // TODO: move the conversion to the server
+            let blocks = res.body
+                .toString('utf8')
+                .trimEnd()
+                .split('\n')
+                .map((line) => JSON.parse(line));
+            return blocks;
+        });
+    }
+    async *stream(query) {
+        let queue = new util_internal_1.AsyncQueue(1);
+        const ingest = async () => {
+            let bufferSize = 0;
+            let fromBlock = query.fromBlock;
+            let toBlock = query.toBlock ?? Infinity;
+            while (fromBlock <= toBlock) {
+                let archiveQuery = { ...query, fromBlock };
+                let res = await this.http
+                    .request('POST', this.getDatasetUrl(`stream`), {
+                    json: archiveQuery,
+                    retryAttempts: 3,
+                    httpTimeout: this.queryTimeout,
+                    stream: true,
+                })
+                    .catch((0, util_internal_1.withErrorContext)({
+                    archiveQuery,
+                }));
+                for await (let lines of (0, util_internal_archive_layout_1.splitLines)(res.body)) {
+                    let batch = queue.peek();
+                    if (batch instanceof Error)
+                        return;
+                    if (!batch) {
+                        bufferSize = 0;
+                    }
+                    let blocks = lines.map((line) => {
+                        bufferSize += line.length;
+                        return JSON.parse(line);
+                    });
+                    if (batch) {
+                        // FIXME: won't it overflow stack?
+                        batch.push(...blocks);
+                        if (bufferSize > this.bufferThreshold) {
+                            await queue.wait();
+                        }
+                    }
+                    else {
+                        await queue.put(blocks);
+                    }
+                    fromBlock = (0, util_internal_1.last)(blocks).header.number + 1;
+                }
+                // no blocks left
+                if (res.status == 204) {
+                    await (0, util_internal_1.wait)(1000);
+                }
+            }
+        };
+        ingest().then(() => queue.close(), (err) => {
+            if (!queue.isClosed()) {
+                queue.forcePut((0, util_internal_1.ensureError)(err));
+            }
+        });
+        for await (let valueOrError of queue.iterate()) {
+            if (valueOrError instanceof Error)
+                throw valueOrError;
+            yield valueOrError;
+        }
+    }
+}
+exports.PortalClient = PortalClient;
+//# sourceMappingURL=client.js.map

package/lib/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";;;;;;AAAA,uDAAgD;AAEhD,2DAA6F;AAC7F,yFAAiE;AACjE,oDAA2B;AA+B3B,MAAa,YAAY;IAMrB,YAAY,OAA4B;QACpC,IAAI,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC/B,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,wBAAU,CAAC,EAAC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAC,CAAC,CAAA;QAC9D,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,MAAO,CAAA;QACnD,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;IACtE,CAAC;IAEO,aAAa,CAAC,IAAY;QAC9B,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACzB,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAClC,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAA;QACtB,CAAC;aAAM,CAAC;YACJ,CAAC,CAAC,QAAQ,IAAI,GAAG,GAAG,IAAI,CAAA;QAC5B,CAAC;QACD,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,SAAS;QACX,IAAI,GAAG,GAAW,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE;YAChE,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,KAAM;SACtB,CAAC,CAAA;QACF,IAAI,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC1B,IAAA,gBAAM,EAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAA;QACpC,OAAO,MAAM,CAAA;IACjB,CAAC;IAED,KAAK,CAAC,WAAW;QACb,IAAI,GAAG,GAAyB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE;YAChF,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,KAAM;SACtB,CAAC,CAAA;QACF,OAAO;YACH,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS;SAC9B,CAAA;IACL,CAAC;IAED,KAAK,CAA+D,KAAQ;QACxE,OAAO,IAAI,CAAC,IAAI;aACX,OAAO,CAAS,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE;YACnD,IAAI,EAAE,KAAK;YACX,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,IAAI,CAAC,YAAY;SACjC,CAAC;aACD,KAAK,CACF,IAAA,gCAAgB,EAAC;YACb,YAAY,EAAE,KAAK;SACtB,CAAC,CACL;aACA,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;YACV,0CAA0C;YAC1C,IAAI,MAAM,GAAG,GAAG,CAAC,IAAI;iBAChB,QAAQ,CAAC,MAAM,CAAC;iBAChB,OAAO,EAAE;iBACT,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;YACpC,OAAO,MAAM,CAAA;QACjB,CAAC,CAAC,CAAA;IACV,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAA+D,KAAQ;QAChF,IAAI,KAAK,GAAG,IAAI,0BAAU,CAAc,CAAC,CAAC,CAAA;QAE1C,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACtB,IAAI,UAAU,GAAG,CAAC,CAAA;YAClB,IAAI,SAAS,GAAG,KAAK,CAAC,SAAS,CAAA;YAC/B,IAAI,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAA;YAEvC,OAAO,SAAS,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,YAAY,GAAG,EAAC,GAAG,KAAK,EAAE,SAAS,EAAC,CAAA;gBAExC,IAAI,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI;qBACpB,OAAO,CAAwB,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE;oBAClE,IAAI,EAAE,YAAY;oBAClB,aAAa,EAAE,CAAC;oBAChB,WAAW,EAAE,IAAI,CAAC,YAAY;oBAC9B,MAAM,EAAE,IAAI;iBACf,CAAC;qBACD,KAAK,CACF,IAAA,gCAAgB,EAAC;oBACb,YAAY;iBACf,CAAC,CACL,CAAA;gBAEL,IAAI,KAAK,EAAE,IAAI,KAAK,IAAI,IAAA,yCAAU,EAAC,GAAG,CAAC,IAA6B,CAAC,EAAE,CAAC;oBACpE,IAAI,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;oBACxB,IAAI,KAAK,YAAY,KAAK;wBAAE,OAAM;oBAElC,IAAI,CAAC,KAAK,EAAE,CAAC;wBACT,UAAU,GAAG,CAAC,CAAA;oBAClB,CAAC;oBAED,IAAI,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;wBAC5B,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;wBACzB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAM,CAAA;oBAChC,CAAC,CAAC,CAAA;oBAEF,IAAI,KAAK,EAAE,CAAC;wBACR,kCAAkC;wBAClC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;wBACrB,IAAI,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;4BACpC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAA;wBACtB,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACJ,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBAC3B,CAAC;oBAED,SAAS,GAAG,IAAA,oBAAI,EAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;gBAC9C,CAAC;gBAED,iBAAiB;gBACjB,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;oBACpB,MAAM,IAAA,oBAAI,EAAC,IAAI,CAAC,CAAA;gBACpB,CAAC;YACL,CAAC;QACL,CAAC,CAAA;QAED,MAAM,EAAE,CAAC,IAAI,CACT,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,EACnB,CAAC,GAAG,EAAE,EAAE;YACJ,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;gBACpB,KAAK,CAAC,QAAQ,CAAC,IAAA,2BAAW,EAAC,GAAG,CAAC,CAAC,CAAA;YACpC,CAAC;QACL,CAAC,CACJ,CAAA;QAED,IAAI,KAAK,EAAE,IAAI,YAAY,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,YAAY,KAAK;gBAAE,MAAM,YAAY,CAAA;YACrD,MAAM,YAAY,CAAA;QACtB,CAAC;IACL,CAAC;CACJ;AAzID,oCAyIC"}

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@subsquid/portal-client",
+  "version": "0.0.1-portal-api.4494d2",
+  "description": "SQD Portal API",
+  "license": "GPL-3.0-or-later",
+  "repository": "git@github.com:subsquid/squid.git",
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "lib/client.js",
+  "files": [
+    "lib",
+    "src"
+  ],
+  "dependencies": {
+    "@subsquid/util-internal": "3.3.0-portal-api.4494d2",
+    "@subsquid/util-internal-range": "^0.3.0",
+    "@subsquid/util-internal-archive-layout": "1.1.0-portal-api.4494d2"
+  },
+  "peerDependencies": {
+    "@subsquid/http-client": "^1.5.0",
+    "@subsquid/logger": "^1.3.3"
+  },
+  "peerDependenciesMeta": {
+    "@subsquid/logger": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@subsquid/http-client": "^1.5.0",
+    "@subsquid/logger": "^1.3.3",
+    "@types/node": "^18.18.14",
+    "typescript": "~5.3.2"
+  },
+  "scripts": {
+    "build": "rm -rf lib && tsc"
+  }
+}

package/src/client.ts

@@ -0,0 +1,173 @@
+import {HttpClient} from '@subsquid/http-client'
+import type {Logger} from '@subsquid/logger'
+import {AsyncQueue, ensureError, last, wait, withErrorContext} from '@subsquid/util-internal'
+import {splitLines} from '@subsquid/util-internal-archive-layout'
+import assert from 'assert'
+
+
+export interface PortalQuery {
+    fromBlock: number
+    toBlock?: number
+}
+
+
+export interface Block {
+    header: {
+        number: number
+        hash: string
+    }
+}
+
+
+export interface Metadata {
+    isRealTime: boolean
+}
+
+
+export interface PortalClientOptions {
+    url: string
+    http?: HttpClient
+    log?: Logger
+    queryTimeout?: number
+    bufferThreshold?: number
+}
+
+
+export class PortalClient {
+    private url: URL
+    private http: HttpClient
+    private queryTimeout: number
+    private bufferThreshold: number
+
+    constructor(options: PortalClientOptions) {
+        this.url = new URL(options.url)
+        this.http = options.http || new HttpClient({log: options.log})
+        this.queryTimeout = options.queryTimeout ?? 180_000
+        this.bufferThreshold = options.bufferThreshold ?? 10 * 1024 * 1024
+    }
+
+    private getDatasetUrl(path: string): string {
+        let u = new URL(this.url)
+        if (this.url.pathname.endsWith('/')) {
+            u.pathname += path
+        } else {
+            u.pathname += '/' + path
+        }
+        return u.toString()
+    }
+
+    async getHeight(): Promise<number> {
+        let res: string = await this.http.get(this.getDatasetUrl('height'), {
+            retryAttempts: 3,
+            httpTimeout: 10_000,
+        })
+        let height = parseInt(res)
+        assert(Number.isSafeInteger(height))
+        return height
+    }
+
+    async getMetadata(): Promise<Metadata> {
+        let res: {real_time: boolean} = await this.http.get(this.getDatasetUrl('metadata'), {
+            retryAttempts: 3,
+            httpTimeout: 10_000,
+        })
+        return {
+            isRealTime: !!res.real_time
+        }
+    }
+
+    query<B extends Block = Block, Q extends PortalQuery = PortalQuery>(query: Q): Promise<B[]> {
+        return this.http
+            .request<Buffer>('POST', this.getDatasetUrl(`stream`), {
+                json: query,
+                retryAttempts: 3,
+                httpTimeout: this.queryTimeout,
+            })
+            .catch(
+                withErrorContext({
+                    archiveQuery: query,
+                })
+            )
+            .then((res) => {
+                // TODO: move the conversion to the server
+                let blocks = res.body
+                    .toString('utf8')
+                    .trimEnd()
+                    .split('\n')
+                    .map((line) => JSON.parse(line))
+                return blocks
+            })
+    }
+
+    async *stream<B extends Block = Block, Q extends PortalQuery = PortalQuery>(query: Q): AsyncIterable<B[]> {
+        let queue = new AsyncQueue<B[] | Error>(1)
+
+        const ingest = async () => {
+            let bufferSize = 0
+            let fromBlock = query.fromBlock
+            let toBlock = query.toBlock ?? Infinity
+
+            while (fromBlock <= toBlock) {
+                let archiveQuery = {...query, fromBlock}
+
+                let res = await this.http
+                    .request<NodeJS.ReadableStream>('POST', this.getDatasetUrl(`stream`), {
+                        json: archiveQuery,
+                        retryAttempts: 3,
+                        httpTimeout: this.queryTimeout,
+                        stream: true,
+                    })
+                    .catch(
+                        withErrorContext({
+                            archiveQuery,
+                        })
+                    )
+
+                for await (let lines of splitLines(res.body as AsyncIterable<Buffer>)) {
+                    let batch = queue.peek()
+                    if (batch instanceof Error) return
+
+                    if (!batch) {
+                        bufferSize = 0
+                    }
+
+                    let blocks = lines.map((line) => {
+                        bufferSize += line.length
+                        return JSON.parse(line) as B
+                    })
+
+                    if (batch) {
+                        // FIXME: won't it overflow stack?
+                        batch.push(...blocks)
+                        if (bufferSize > this.bufferThreshold) {
+                            await queue.wait()
+                        }
+                    } else {
+                        await queue.put(blocks)
+                    }
+
+                    fromBlock = last(blocks).header.number + 1
+                }
+
+                // no blocks left
+                if (res.status == 204) {
+                    await wait(1000)
+                }
+            }
+        }
+
+        ingest().then(
+            () => queue.close(),
+            (err) => {
+                if (!queue.isClosed()) {
+                    queue.forcePut(ensureError(err))
+                }
+            }
+        )
+
+        for await (let valueOrError of queue.iterate()) {
+            if (valueOrError instanceof Error) throw valueOrError
+            yield valueOrError
+        }
+    }
+}