@subsquid/portal-client 0.3.2 → 0.4.0

package/lib/client.d.ts

@@ -1,5 +1,5 @@
 import { HttpClient, HttpClientOptions } from '@subsquid/http-client';
-import { GetBlock, evm, PortalBlock, PortalQuery, Query, solana, substrate } from './query';
+import { type AnyQuery, type GetQueryBlock } from './query';
 export * from './query';
 export interface PortalClientOptions {
     /**
@@ -12,17 +12,17 @@ export interface PortalClientOptions {
     http?: HttpClient | HttpClientOptions;
     /**
      * Minimum number of bytes to return.
-     * @default 10_485_760 (10MB)
+     * @deprecated not used
      */
     minBytes?: number;
     /**
      * Maximum number of bytes to return.
-     * @default minBytes
+     * @default 52_428_800 (50MB)
      */
     maxBytes?: number;
     /**
      * Maximum time between stream data in milliseconds for return.
-     * @default 300
+     * @default 500
      */
     maxIdleTime?: number;
     /**
@@ -46,16 +46,34 @@ export interface PortalRequestOptions {
 }
 export interface PortalStreamOptions {
     request?: Omit<PortalRequestOptions, 'abort'>;
-    minBytes?: number;
     maxBytes?: number;
     maxIdleTime?: number;
     maxWaitTime?: number;
     headPollInterval?: number;
 }
+export interface PortalStreamBaseResponse {
+    headNumber?: number;
+    finalizedHeadNumber?: number;
+    finalizedHeadHash?: string;
+}
+export interface PortalStreamDataResponse extends PortalStreamBaseResponse {
+    type: 'data';
+    data: AsyncIterable<Uint8Array> | null;
+}
+export interface PortalStreamNoDataResponse extends PortalStreamBaseResponse {
+    type: 'no-data';
+}
+export interface PortalStreamForkResponse extends PortalStreamBaseResponse {
+    type: 'fork';
+    previousBlocks: BlockRef[];
+}
+export type PortalStreamResponse = PortalStreamDataResponse | PortalStreamNoDataResponse | PortalStreamForkResponse;
 export type PortalStreamData<B> = {
     blocks: B[];
-    finalizedHead?: BlockRef;
     meta: {
+        finalizedHeadNumber?: number;
+        finalizedHeadHash?: string;
+        headNumber?: number;
         bytes: number;
     };
 };
@@ -69,31 +87,35 @@ export declare class PortalClient {
     private url;
     private client;
     private headPollInterval;
-    private minBytes;
     private maxBytes;
     private maxIdleTime;
     private maxWaitTime;
     constructor(options: PortalClientOptions);
     private getDatasetUrl;
-    getHead(options?: PortalRequestOptions): Promise<BlockRef | undefined>;
+    getHead(options?: PortalRequestOptions, finalized?: boolean): Promise<BlockRef | undefined>;
     getFinalizedHead(options?: PortalRequestOptions): Promise<BlockRef | undefined>;
     /**
-     * @deprecated
+     * Waits until the finalized head reaches or exceeds the specified block number.
+     * @param blockNumber - The block number to wait for finalization
+     * @param options - Request options including abort signal and poll interval
+     * @returns The finalized head once it reaches the target block number
      */
-    getFinalizedHeight(options?: PortalRequestOptions): Promise<number>;
-    getFinalizedQuery<Q extends Query>(query: Q, options?: PortalRequestOptions): Promise<GetBlock<Q>[]>;
-    getQuery<Q extends PortalQuery = PortalQuery, R extends PortalBlock = PortalBlock>(query: Q, options?: PortalRequestOptions): Promise<R[]>;
-    getFinalizedStream<Q extends evm.Query | solana.Query | substrate.Query>(query: Q, options?: PortalStreamOptions): PortalStream<GetBlock<Q>>;
-    getStream<Q extends evm.Query | solana.Query | substrate.Query>(query: Q, options?: PortalStreamOptions): PortalStream<GetBlock<Q>>;
+    waitForFinalization(blockNumber: number, options?: PortalRequestOptions & {
+        pollInterval?: number;
+    }): Promise<BlockRef>;
+    getStream<Q extends AnyQuery>(query: Q, options?: PortalStreamOptions, finalized?: boolean): PortalStream<GetQueryBlock<Q>>;
+    getFinalizedStream<Q extends AnyQuery>(query: Q, options?: PortalStreamOptions): PortalStream<GetQueryBlock<Q>>;
     private getStreamOptions;
     private getStreamRequest;
     private request;
 }
 export declare class ForkException extends Error {
-    readonly previousBlocks: BlockRef[];
-    readonly head: BlockRef;
-    readonly name = "ForkError";
-    constructor(previousBlocks: BlockRef[], head: BlockRef);
+    blockNumber: number;
+    parentBlockHash: string;
+    previousBlocks: BlockRef[];
+    constructor(blockNumber: number, parentBlockHash: string, previousBlocks: BlockRef[]);
+    get name(): string;
+    get isSubsquidForkException(): true;
 }
 export declare function isForkException(err: unknown): err is ForkException;
 //# sourceMappingURL=client.d.ts.map

package/lib/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,UAAU,EAAE,iBAAiB,EAA0C,MAAM,uBAAuB,CAAA;AAWtH,OAAO,EAAC,QAAQ,EAAE,GAAG,EAAkB,WAAW,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAC,MAAM,SAAS,CAAA;AAGzG,cAAc,SAAS,CAAA;AAIvB,MAAM,WAAW,mBAAmB;IAChC;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IAEX;;OAEG;IACH,IAAI,CAAC,EAAE,UAAU,GAAG,iBAAiB,CAAA;IAErC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,CAAC,EAAE,WAAW,CAAA;IACrB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;IACxB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,WAAW,CAAA;CACtB;AAED,MAAM,WAAW,mBAAmB;IAChC,OAAO,CAAC,EAAE,IAAI,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAA;IAE7C,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAAI;IAC9B,MAAM,EAAE,CAAC,EAAE,CAAA;IACX,aAAa,CAAC,EAAE,QAAQ,CAAA;IACxB,IAAI,EAAE;QACF,KAAK,EAAE,MAAM,CAAA;KAChB,CAAA;CACJ,CAAA;AAED,MAAM,WAAW,YAAY,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;CAAG;AAE9E,MAAM,MAAM,QAAQ,GAAG;IACnB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;CACjB,CAAA;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,GAAG,CAAK;IAChB,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,gBAAgB,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,WAAW,CAAQ;gBAEf,OAAO,EAAE,mBAAmB;IAUxC,OAAO,CAAC,aAAa;IAUf,OAAO,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAKtE,gBAAgB,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAKrF;;OAEG;IACG,kBAAkB,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,MAAM,CAAC;IAMzE,iBAAiB,CAAC,CAAC,SAAS,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IAqBpG,QAAQ,CAAC,CAAC,SAAS,WAAW,GAAG,WAAW,EAAE,CAAC,SAAS,WAAW,GAAG,WAAW,EAC7E,KAAK,EAAE,CAAC,EACR,OAAO,CAAC,EAAE,oBAAoB,GAC/B,OAAO,CAAC,CAAC,EAAE,CAAC;IAqBf,kBAAkB,CAAC,CAAC,SAAS,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,EACnE,KAAK,EAAE,CAAC,EACR,OAAO,CAAC,EAAE,mBAAmB,GAC9B,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAM5B,SAAS,CAAC,CAAC,SAAS,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,EAC1D,KAAK,EAAE,CAAC,EACR,OAAO,CAAC,EAAE,mBAAmB,GAC9B,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAM5B,OAAO,CAAC,gBAAgB;YAoBV,gBAAgB;IA0C9B,OAAO,CAAC,OAAO;CASlB;AAySD,qBAAa,aAAc,SAAQ,KAAK;IAGxB,QAAQ,CAAC,cAAc,EAAE,QAAQ,EAAE;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ;IAFxE,QAAQ,CAAC,IAAI,eAAc;gBAEN,cAAc,EAAE,QAAQ,EAAE,EAAW,IAAI,EAAE,QAAQ;CAQ3E;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,GAAG,IAAI,aAAa,CAIlE"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,UAAU,EAAE,iBAAiB,EAA0C,MAAM,uBAAuB,CAAA;AAItH,OAAO,EAAW,KAAK,QAAQ,EAAE,KAAK,aAAa,EAAC,MAAM,SAAS,CAAA;AAEnE,cAAc,SAAS,CAAA;AAIvB,MAAM,WAAW,mBAAmB;IAChC;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IAEX;;OAEG;IACH,IAAI,CAAC,EAAE,UAAU,GAAG,iBAAiB,CAAA;IAErC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,CAAC,EAAE,WAAW,CAAA;IACrB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;IACxB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,WAAW,CAAA;CACtB;AAED,MAAM,WAAW,mBAAmB;IAChC,OAAO,CAAC,EAAE,IAAI,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAA;IAE7C,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,wBAAwB;IACrC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAC7B;AAED,MAAM,WAAW,wBAAyB,SAAQ,wBAAwB;IACtE,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,IAAI,CAAA;CACzC;AAED,MAAM,WAAW,0BAA2B,SAAQ,wBAAwB;IACxE,IAAI,EAAE,SAAS,CAAA;CAClB;AAED,MAAM,WAAW,wBAAyB,SAAQ,wBAAwB;IACtE,IAAI,EAAE,MAAM,CAAA;IACZ,cAAc,EAAE,QAAQ,EAAE,CAAA;CAC7B;AAED,MAAM,MAAM,oBAAoB,GAAG,wBAAwB,GAAG,0BAA0B,GAAG,wBAAwB,CAAA;AAEnH,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAAI;IAC9B,MAAM,EAAE,CAAC,EAAE,CAAA;IACX,IAAI,EAAE;QACF,mBAAmB,CAAC,EAAE,MAAM,CAAA;QAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAA;QAC1B,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,KAAK,EAAE,MAAM,CAAA;KAChB,CAAA;CACJ,CAAA;AAED,MAAM,WAAW,YAAY,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;CAAG;AAE9E,MAAM,MAAM,QAAQ,GAAG;IACnB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;CACjB,CAAA;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,GAAG,CAAK;IAChB,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,gBAAgB,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,WAAW,CAAQ;gBAEf,OAAO,EAAE,mBAAmB;IASxC,OAAO,CAAC,aAAa;IAUf,OAAO,CAAC,OAAO,CAAC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAKjG,gBAAgB,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAI/E;;;;;OAKG;IACG,mBAAmB,CACrB,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,oBAAoB,GAAG;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAC,GACzD,OAAO,CAAC,QAAQ,CAAC;IAepB,SAAS,CAAC,CAAC,SAAS,QAAQ,EACxB,KAAK,EAAE,CAAC,EACR,OAAO,CAAC,EAAE,mBAAmB,EAC7B,SAAS,CAAC,EAAE,OAAO,GACpB,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAMjC,kBAAkB,CAAC,CAAC,SAAS,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAI/G,OAAO,CAAC,gBAAgB;YAkBV,gBAAgB;IA+C9B,OAAO,CAAC,OAAO;CASlB;AAuSD,qBAAa,aAAc,SAAQ,KAAK;IACjB,WAAW,EAAE,MAAM;IAAS,eAAe,EAAE,MAAM;IAAS,cAAc,EAAE,QAAQ,EAAE;gBAAtF,WAAW,EAAE,MAAM,EAAS,eAAe,EAAE,MAAM,EAAS,cAAc,EAAE,QAAQ,EAAE;IAQzG,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,IAAI,uBAAuB,IAAI,IAAI,CAElC;CACJ;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,GAAG,IAAI,aAAa,CAIlE"}

package/lib/client.js

@@ -18,8 +18,9 @@ exports.ForkException = exports.PortalClient = void 0;
 exports.isForkException = isForkException;
 const http_client_1 = require("@subsquid/http-client");
 const util_internal_1 = require("@subsquid/util-internal");
-const query_1 = require("./query");
 const util_internal_validation_1 = require("@subsquid/util-internal-validation");
+const util_1 = require("./util");
+const query_1 = require("./query");
 __exportStar(require("./query"), exports);
 const USER_AGENT = `@subsquid/portal-client (https://sqd.ai)`;
 class PortalClient {
@@ -27,9 +28,8 @@ class PortalClient {
         this.url = new URL(options.url);
         this.client = options.http instanceof http_client_1.HttpClient ? options.http : new http_client_1.HttpClient(options.http);
         this.headPollInterval = options.headPollInterval ?? 0;
-        this.minBytes = options.minBytes ?? 10 * 1024 * 1024;
-        this.maxBytes = options.maxBytes ?? this.minBytes;
-        this.maxIdleTime = options.maxIdleTime ?? 300;
+        this.maxBytes = options.maxBytes ?? 50 * 1024 * 1024;
+        this.maxIdleTime = options.maxIdleTime ?? 500;
         this.maxWaitTime = options.maxWaitTime ?? 5000;
     }
     getDatasetUrl(path) {
@@ -42,69 +42,40 @@ class PortalClient {
         }
         return u.toString();
     }
-    async getHead(options) {
-        const res = await this.request('GET', this.getDatasetUrl('head'), options);
+    async getHead(options, finalized) {
+        const res = await this.request('GET', this.getDatasetUrl(!finalized ? 'head' : 'finalized-head'), options);
         return res.body ?? undefined;
     }
-    async getFinalizedHead(options) {
-        const res = await this.request('GET', this.getDatasetUrl('finalized-head'), options);
-        return res.body ?? undefined;
+    getFinalizedHead(options) {
+        return this.getHead(options, true);
     }
     /**
-     * @deprecated
+     * Waits until the finalized head reaches or exceeds the specified block number.
+     * @param blockNumber - The block number to wait for finalization
+     * @param options - Request options including abort signal and poll interval
+     * @returns The finalized head once it reaches the target block number
      */
-    async getFinalizedHeight(options) {
-        let { body } = await this.request('GET', this.getDatasetUrl('finalized-stream/height'), options);
-        let height = parseInt(body);
-        return height;
-    }
-    getFinalizedQuery(query, options) {
-        // FIXME: is it needed or it is better to always use stream?
-        return this.request('POST', this.getDatasetUrl(`finalized-stream`), {
-            ...options,
-            json: query,
-        })
-            .catch((0, util_internal_1.withErrorContext)({
-            archiveQuery: query,
-        }))
-            .then((res) => {
-            let blocks = res.body
-                .toString('utf8')
-                .trimEnd()
-                .split('\n')
-                .map((line) => JSON.parse(line));
-            return blocks;
-        });
+    async waitForFinalization(blockNumber, options) {
+        let { pollInterval = this.headPollInterval || 1000, ...requestOptions } = options ?? {};
+        while (true) {
+            options?.abort?.throwIfAborted();
+            let head = await this.getFinalizedHead(requestOptions);
+            if (head && head.number >= blockNumber) {
+                return head;
+            }
+            await (0, util_internal_1.wait)(pollInterval, options?.abort);
+        }
     }
-    getQuery(query, options) {
-        // FIXME: is it needed or it is better to always use stream?
-        return this.request('POST', this.getDatasetUrl(`stream`), {
-            ...options,
-            json: query,
-        })
-            .catch((0, util_internal_1.withErrorContext)({
-            archiveQuery: query,
-        }))
-            .then((res) => {
-            let blocks = res.body
-                .toString('utf8')
-                .trimEnd()
-                .split('\n')
-                .map((line) => JSON.parse(line));
-            return blocks;
-        });
+    getStream(query, options, finalized) {
+        return createPortalStream(query, this.getStreamOptions(options), async (q, o) => this.getStreamRequest(!finalized ? 'stream' : 'finalized-stream', q, o));
     }
     getFinalizedStream(query, options) {
-        return createPortalStream(query, this.getStreamOptions(options), async (q, o) => this.getStreamRequest('finalized-stream', q, o));
-    }
-    getStream(query, options) {
-        return createPortalStream(query, this.getStreamOptions(options), async (q, o) => this.getStreamRequest('stream', q, o));
+        return this.getStream(query, options, true);
     }
     getStreamOptions(options) {
-        let { headPollInterval = this.headPollInterval, minBytes = this.minBytes, maxBytes = this.maxBytes, maxIdleTime = this.maxIdleTime, maxWaitTime = this.maxWaitTime, request = {}, } = options ?? {};
+        let { headPollInterval = this.headPollInterval, maxBytes = this.maxBytes, maxIdleTime = this.maxIdleTime, maxWaitTime = this.maxWaitTime, request = {}, } = options ?? {};
         return {
             headPollInterval,
-            minBytes,
             maxBytes,
             maxIdleTime,
             maxWaitTime,
@@ -120,28 +91,29 @@ class PortalClient {
             }).catch((0, util_internal_1.withErrorContext)({
                 query: query,
             }));
+            let headers = getPortalStreamHeaders(res.headers);
             switch (res.status) {
                 case 200:
-                    let finalizedHead = getFinalizedHeadHeader(res.headers);
-                    let stream = res.body ? splitLines(res.body) : undefined;
                     return {
-                        finalizedHead,
-                        stream,
+                        ...headers,
+                        type: 'data',
+                        data: res.body ?? null,
                     };
                 case 204:
                     return {
-                        finalizedHead: getFinalizedHeadHeader(res.headers),
+                        ...headers,
+                        type: 'no-data',
                     };
                 default:
                     throw (0, util_internal_1.unexpectedCase)(res.status);
             }
         }
         catch (e) {
-            if (isForkHttpError(e) && query.fromBlock != null && query.parentBlockHash != null) {
-                e = new ForkException(e.response.body.previousBlocks, {
-                    number: query.fromBlock - 1,
-                    hash: query.parentBlockHash,
-                });
+            if (isForkHttpError(e)) {
+                return {
+                    type: 'fork',
+                    previousBlocks: e.response.body.previousBlocks,
+                };
             }
             throw (0, util_internal_1.addErrorContext)(e, {
                 query,
@@ -171,35 +143,35 @@ function isForkHttpError(err) {
 function createPortalStream(query, options, requestStream) {
     let { headPollInterval, request, ...bufferOptions } = options;
     let buffer = new PortalStreamBuffer(bufferOptions);
-    let schema = (0, query_1.getBlockSchema)(query);
-    let { fromBlock = 0, toBlock, parentBlockHash } = query;
+    let [normalizedQuery, schema] = (0, query_1.getQuery)(query);
+    let { fromBlock = 0, toBlock, parentBlockHash } = normalizedQuery;
     const ingest = async () => {
         while (!buffer.signal.aborted) {
             if (toBlock != null && fromBlock > toBlock)
                 break;
             let res = await requestStream({
-                ...query,
+                ...normalizedQuery,
                 fromBlock,
                 parentBlockHash,
             }, {
                 ...request,
                 abort: buffer.signal,
             });
-            const finalizedHead = res.finalizedHead;
-            // we are on head
-            if (!('stream' in res)) {
-                await buffer.put({ blocks: [], meta: { bytes: 0 }, finalizedHead });
+            if (res.type === 'fork') {
+                throw new ForkException(fromBlock, parentBlockHash, res.previousBlocks);
+            }
+            buffer.updateHead(res);
+            if (res.type === 'no-data') {
                 buffer.flush();
                 if (headPollInterval > 0) {
                     await (0, util_internal_1.wait)(headPollInterval, buffer.signal);
                 }
                 continue;
             }
-            // no data left on this range
-            if (res.stream == null)
+            if (!res.data)
                 break;
             try {
-                for await (let data of res.stream) {
+                for await (let data of splitLines(res.data)) {
                     buffer.signal.throwIfAborted();
                     let blocks = [];
                     let bytes = 0;
@@ -210,8 +182,9 @@ function createPortalStream(query, options, requestStream) {
                         fromBlock = block.header.number + 1;
                         parentBlockHash = block.header.hash;
                     }
-                    await buffer.put({ blocks, finalizedHead, meta: { bytes } });
+                    await buffer.put(blocks, bytes);
                 }
+                buffer.flush();
             }
             catch (err) {
                 if (buffer.signal.aborted)
@@ -221,141 +194,139 @@ function createPortalStream(query, options, requestStream) {
                 }
             }
         }
+        buffer.flush();
     };
     ingest().then(() => buffer.close(), (err) => buffer.fail(err));
-    return buffer.iterate();
+    return buffer;
 }
 class PortalStreamBuffer {
     get signal() {
         return this.abortController.signal;
     }
     constructor(options) {
-        this.state = 'pending';
-        this.readyFuture = (0, util_internal_1.createFuture)();
-        this.takeFuture = (0, util_internal_1.createFuture)();
-        this.putFuture = (0, util_internal_1.createFuture)();
+        this.blocks = [];
+        this.bytes = 0;
+        this.headUpdated = false;
+        this.state = 'open';
+        this.waitTimeouted = false;
+        this.putMutex = new util_1.Semaphore(false);
+        this.takeMutex = new util_1.Semaphore(true);
         this.abortController = new AbortController();
-        this.maxWaitTime = options.maxWaitTime;
-        this.minBytes = options.minBytes;
-        this.maxBytes = Math.max(options.maxBytes, options.minBytes);
-        this.maxIdleTime = options.maxIdleTime;
+        this.maxBytes = options.maxBytes;
+        this.idleTimer = new util_1.Timer(options.maxIdleTime, () => this.flush());
+        this.waitTimer = new util_1.Timer(options.maxWaitTime, () => (this.waitTimeouted = true));
     }
     async take() {
-        if (this.state === 'pending') {
-            this.waitTimeout = setTimeout(() => this._ready('wait'), this.maxWaitTime);
-        }
-        await Promise.all([this.readyFuture.promise(), this.putFuture.promise()]);
-        if (this.state === 'failed') {
-            throw this.error;
-        }
-        let result = this.buffer;
-        this.buffer = undefined;
-        if (this.state === 'closed') {
-            return result;
-        }
-        if (result == null) {
-            throw new Error('Buffer is empty');
+        if (this.isTerminated()) {
+            return this.collect();
         }
-        this.takeFuture.resolve();
-        this.readyFuture = (0, util_internal_1.createFuture)();
-        this.putFuture = (0, util_internal_1.createFuture)();
-        this.takeFuture = (0, util_internal_1.createFuture)();
-        this.state = 'pending';
+        this.waitTimer.start();
+        await this.putMutex.wait();
+        this.waitTimer.stop();
+        let result = this.collect();
+        this.putMutex.unready();
+        this.takeMutex.ready();
+        this.waitTimeouted = false;
         return result;
     }
-    async put(data) {
-        if (this.state === 'closed' || this.state === 'failed') {
+    updateHead(head) {
+        this.headNumber = head.headNumber;
+        this.finalizedHeadNumber = head.finalizedHeadNumber;
+        this.finalizedHeadHash = head.finalizedHeadHash;
+        this.headUpdated = true;
+    }
+    async put(blocks, bytes) {
+        if (this.isTerminated()) {
             throw new Error('Buffer is closed');
         }
-        if (this.idleTimeout != null) {
-            clearTimeout(this.idleTimeout);
-            this.idleTimeout = undefined;
-        }
-        if (this.buffer == null) {
-            this.buffer = { blocks: [], meta: { bytes: 0 } };
-        }
-        this.buffer.blocks.push(...data.blocks);
-        this.buffer.finalizedHead = data.finalizedHead;
-        this.buffer.meta.bytes += data.meta.bytes;
-        this.putFuture.resolve();
-        if (this.buffer.meta.bytes >= this.minBytes) {
-            this._ready('minBytes');
+        this.idleTimer.stop();
+        this.blocks.push(...blocks);
+        this.bytes += bytes;
+        if (this.waitTimeouted || this.bytes >= this.maxBytes) {
+            this.flush();
         }
-        if (this.buffer.meta.bytes >= this.maxBytes) {
-            await this.takeFuture.promise();
-        }
-        if (this.state === 'pending') {
-            this.idleTimeout = setTimeout(() => this._ready('idle'), this.maxIdleTime);
+        if (this.bytes >= this.maxBytes) {
+            this.takeMutex.unready();
+            await this.takeMutex.wait();
         }
+        this.idleTimer.start();
     }
     flush() {
-        if (this.buffer == null)
+        if (!this.hasData())
             return;
-        this._ready('flush');
+        this.stopTimers();
+        this.putMutex.ready();
     }
     close() {
-        if (this.state === 'closed' || this.state === 'failed')
+        if (this.isTerminated())
             return;
         this.state = 'closed';
-        this._cleanup();
+        this.stopTimers();
+        this.putMutex.ready();
+        this.takeMutex.reject(new Error('Buffer closed'));
+        this.abortController.abort();
     }
     fail(err) {
-        if (this.state === 'closed' || this.state === 'failed')
+        if (this.isTerminated())
             return;
         this.state = 'failed';
         this.error = err;
-        this._cleanup();
+        this.stopTimers();
+        this.putMutex.ready();
+        this.takeMutex.reject(new Error('Buffer closed'));
+        this.abortController.abort();
     }
-    iterate() {
+    [Symbol.asyncIterator]() {
         return {
-            [Symbol.asyncIterator]: () => {
-                return {
-                    next: async () => {
-                        const value = await this.take();
-                        if (value == null) {
-                            return { done: true, value: undefined };
-                        }
-                        return { done: false, value };
-                    },
-                    return: async () => {
-                        this.close();
-                        return { done: true, value: undefined };
-                    },
-                    throw: async (error) => {
-                        this.fail(error);
-                        throw error;
-                    },
-                };
+            next: async () => {
+                let value = await this.take();
+                if (value == null) {
+                    return { done: true, value: undefined };
+                }
+                return { done: false, value };
+            },
+            return: async () => {
+                this.close();
+                return { done: true, value: undefined };
+            },
+            throw: async (error) => {
+                this.fail(error);
+                throw error;
             },
         };
     }
-    _ready(reason) {
-        if (this.state === 'pending') {
-            this.state = 'ready';
-            this.readyFuture.resolve();
-        }
-        if (this.idleTimeout != null) {
-            clearTimeout(this.idleTimeout);
-            this.idleTimeout = undefined;
-        }
-        if (this.waitTimeout != null) {
-            clearTimeout(this.waitTimeout);
-            this.waitTimeout = undefined;
-        }
+    hasData() {
+        return this.blocks.length > 0 || this.headUpdated;
     }
-    _cleanup() {
-        if (this.idleTimeout != null) {
-            clearTimeout(this.idleTimeout);
-            this.idleTimeout = undefined;
-        }
-        if (this.waitTimeout != null) {
-            clearTimeout(this.waitTimeout);
-            this.waitTimeout = undefined;
+    isTerminated() {
+        return this.state !== 'open';
+    }
+    isFailed() {
+        return this.state === 'failed';
+    }
+    collect() {
+        if (!this.hasData()) {
+            if (this.isFailed())
+                throw this.error;
+            return undefined;
         }
-        this.readyFuture.resolve();
-        this.putFuture.resolve();
-        this.takeFuture.resolve();
-        this.abortController.abort();
+        let result = {
+            blocks: this.blocks,
+            meta: {
+                bytes: this.bytes,
+                headNumber: this.headNumber,
+                finalizedHeadNumber: this.finalizedHeadNumber,
+                finalizedHeadHash: this.finalizedHeadHash,
+            },
+        };
+        this.blocks = [];
+        this.bytes = 0;
+        this.headUpdated = false;
+        return result;
+    }
+    stopTimers() {
+        this.idleTimer.stop();
+        this.waitTimer.stop();
     }
 }
 async function* splitLines(chunks) {
@@ -402,12 +373,19 @@ class LineSplitter {
     }
 }
 class ForkException extends Error {
-    constructor(previousBlocks, head) {
-        let parent = (0, util_internal_1.last)(previousBlocks);
-        super(`expected ${head.number + 1} to have parent ${parent.number}#${parent.hash}, but got ${head.number}#${head.hash}`);
+    constructor(blockNumber, parentBlockHash, previousBlocks) {
+        let base = (0, util_internal_1.last)(previousBlocks);
+        super(`expected block ${blockNumber} to have parent ${base.number}#${parentBlockHash}, ` +
+            `but got ${base.number}#${base.hash} as a parent instead`);
+        this.blockNumber = blockNumber;
+        this.parentBlockHash = parentBlockHash;
         this.previousBlocks = previousBlocks;
-        this.head = head;
-        this.name = 'ForkError';
+    }
+    get name() {
+        return 'ForkException';
+    }
+    get isSubsquidForkException() {
+        return true;
     }
 }
 exports.ForkException = ForkException;
@@ -418,15 +396,15 @@ function isForkException(err) {
         return true;
     return false;
 }
-function getFinalizedHeadHeader(headers) {
-    let finalizedHeadHash = headers.get('X-Sqd-Finalized-Head-Hash');
-    let finalizedHeadNumber = headers.get('X-Sqd-Finalized-Head-Number');
-    return finalizedHeadHash != null && finalizedHeadNumber != null
-        ? {
-            hash: finalizedHeadHash,
-            number: parseInt(finalizedHeadNumber),
-        }
-        : undefined;
+function getPortalStreamHeaders(headers) {
+    let finalizedHeadHash = headers.get('x-sqd-finalized-head-hash');
+    let finalizedHeadNumber = headers.get('x-sqd-finalized-head-number');
+    let headNumber = headers.get('x-sqd-head-number');
+    return {
+        headNumber: headNumber ? parseInt(headNumber) : undefined,
+        finalizedHeadNumber: finalizedHeadNumber ? parseInt(finalizedHeadNumber) : undefined,
+        finalizedHeadHash: finalizedHeadHash ?? undefined,
+    };
 }
 function isStreamAbortedError(err) {
     if (!(err instanceof Error))