@subsquid/batch-processor 0.0.0 → 1.0.0-portal-api.d887ad

package/lib/database.d.ts

@@ -10,9 +10,12 @@ export interface FinalTxInfo {
 }
 export interface FinalDatabase<S> {
     supportsHotBlocks?: false;
-    connect(): Promise<HashAndHeight>;
+    connect(): Promise<FinalDatabaseState>;
+    getState?(): Promise<FinalDatabaseState>;
     transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>;
 }
+export interface FinalDatabaseState extends HashAndHeight {
+}
 export interface HotTxInfo {
     finalizedHead: HashAndHeight;
     baseHead: HashAndHeight;
@@ -21,12 +24,9 @@ export interface HotTxInfo {
 export interface HotDatabase<S> {
     supportsHotBlocks: true;
     connect(): Promise<HotDatabaseState>;
+    getState(): Promise<HotDatabaseState>;
     transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>;
-    /**
-     * @deprecated
-     */
-    transactHot(info: HotTxInfo, cb: (store: S, block: HashAndHeight) => Promise<void>): Promise<void>;
-    transactHot2?(info: HotTxInfo, cb: (store: S, blockSliceStart: number, blockSliceEnd: number) => Promise<void>): Promise<void>;
+    transactHot2(info: HotTxInfo, cb: (store: S, blockSliceStart: number, blockSliceEnd: number) => Promise<void>): Promise<void>;
 }
 export interface HotDatabaseState extends HashAndHeight {
     top: HashAndHeight[];

package/lib/database.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../src/database.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;AAG3D,MAAM,WAAW,WAAW;IACxB,QAAQ,EAAE,aAAa,CAAA;IACvB,QAAQ,EAAE,aAAa,CAAA;IACvB,OAAO,EAAE,OAAO,CAAA;CACnB;AAGD,MAAM,WAAW,aAAa,CAAC,CAAC;IAC5B,iBAAiB,CAAC,EAAE,KAAK,CAAA;IACzB,OAAO,IAAI,OAAO,CAAC,aAAa,CAAC,CAAA;IACjC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9E;AAGD,MAAM,WAAW,SAAS;IACtB,aAAa,EAAE,aAAa,CAAA;IAC5B,QAAQ,EAAE,aAAa,CAAA;IACvB,SAAS,EAAE,aAAa,EAAE,CAAA;CAC7B;AAGD,MAAM,WAAW,WAAW,CAAC,CAAC;IAC1B,iBAAiB,EAAE,IAAI,CAAA;IACvB,OAAO,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAA;IACpC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC3E;;OAEG;IACH,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAElG,YAAY,CAAC,CACT,IAAI,EAAE,SAAS,EACf,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,eAAe,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAChF,OAAO,CAAC,IAAI,CAAC,CAAA;CACnB;AAGD,MAAM,WAAW,gBAAiB,SAAQ,aAAa;IACnD,GAAG,EAAE,aAAa,EAAE,CAAA;CACvB;AAGD,MAAM,WAAW,aAAa;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACf"}
+{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../src/database.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;AAG3D,MAAM,WAAW,WAAW;IACxB,QAAQ,EAAE,aAAa,CAAA;IACvB,QAAQ,EAAE,aAAa,CAAA;IACvB,OAAO,EAAE,OAAO,CAAA;CACnB;AAGD,MAAM,WAAW,aAAa,CAAC,CAAC;IAC5B,iBAAiB,CAAC,EAAE,KAAK,CAAA;IACzB,OAAO,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAAA;IACtC,QAAQ,CAAC,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAAA;IACxC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9E;AAGD,MAAM,WAAW,kBAAmB,SAAQ,aAAa;CAAG;AAG5D,MAAM,WAAW,SAAS;IACtB,aAAa,EAAE,aAAa,CAAA;IAC5B,QAAQ,EAAE,aAAa,CAAA;IACvB,SAAS,EAAE,aAAa,EAAE,CAAA;CAC7B;AAGD,MAAM,WAAW,WAAW,CAAC,CAAC;IAC1B,iBAAiB,EAAE,IAAI,CAAA;IACvB,OAAO,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAA;IACpC,QAAQ,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAE3E,YAAY,CACR,IAAI,EAAE,SAAS,EACf,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,eAAe,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAChF,OAAO,CAAC,IAAI,CAAC,CAAA;CACnB;AAGD,MAAM,WAAW,gBAAiB,SAAQ,aAAa;IACnD,GAAG,EAAE,aAAa,EAAE,CAAA;CACvB;AAGD,MAAM,WAAW,aAAa;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACf"}

package/lib/index.d.ts

@@ -1,4 +1,3 @@
 export * from './database';
-export * from './datasource';
 export * from './run';
 //# sourceMappingURL=index.d.ts.map

package/lib/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAA;AAC1B,cAAc,cAAc,CAAA;AAC5B,cAAc,OAAO,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAA;AAC1B,cAAc,OAAO,CAAA"}

package/lib/index.js

@@ -15,6 +15,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./database"), exports);
-__exportStar(require("./datasource"), exports);
 __exportStar(require("./run"), exports);
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6CAA0B;AAC1B,+CAA4B;AAC5B,wCAAqB"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6CAA0B;AAC1B,wCAAqB"}

package/lib/run.d.ts

@@ -1,5 +1,5 @@
-import { Database, HashAndHeight } from './database';
-import { DataSource } from './datasource';
+import { Database } from './database';
+import { DataSource, BlockRef } from '@subsquid/util-internal-data-source';
 export interface DataHandlerContext<Block, Store> {
     /**
      * Storage interface provided by the database
@@ -14,8 +14,8 @@ export interface DataHandlerContext<Block, Store> {
      */
     isHead: boolean;
 }
-interface BlockBase {
-    header: HashAndHeight;
+export interface BlockBase {
+    header: BlockRef;
 }
 /**
  * Run data processing.
@@ -32,5 +32,4 @@ interface BlockBase {
  * @param dataHandler - The data handler, see {@link DataHandlerContext} for an API available to the handler.
  */
 export declare function run<Block extends BlockBase, Store>(src: DataSource<Block>, db: Database<Store>, dataHandler: (ctx: DataHandlerContext<Block, Store>) => Promise<void>): void;
-export {};
 //# sourceMappingURL=run.d.ts.map

package/lib/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,QAAQ,EAAE,aAAa,EAAC,MAAM,YAAY,CAAA;AAClD,OAAO,EAAC,UAAU,EAAC,MAAM,cAAc,CAAA;AAQvC,MAAM,WAAW,kBAAkB,CAAC,KAAK,EAAE,KAAK;IAC5C;;OAEG;IACH,KAAK,EAAE,KAAK,CAAA;IACZ;;OAEG;IACH,MAAM,EAAE,KAAK,EAAE,CAAA;IACf;;OAEG;IACH,MAAM,EAAE,OAAO,CAAA;CAClB;AAGD,UAAU,SAAS;IACf,MAAM,EAAE,aAAa,CAAA;CACxB;AAGD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,GAAG,CAAC,KAAK,SAAS,SAAS,EAAE,KAAK,EAC9C,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,EACtB,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,EACnB,WAAW,EAAE,CAAC,GAAG,EAAE,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GACtE,IAAI,CAMN"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAgB,QAAQ,EAAmB,MAAM,YAAY,CAAA;AAEpE,OAAO,EAAC,UAAU,EAAmB,QAAQ,EAAC,MAAM,qCAAqC,CAAA;AAMzF,MAAM,WAAW,kBAAkB,CAAC,KAAK,EAAE,KAAK;IAC5C;;OAEG;IACH,KAAK,EAAE,KAAK,CAAA;IACZ;;OAEG;IACH,MAAM,EAAE,KAAK,EAAE,CAAA;IACf;;OAEG;IACH,MAAM,EAAE,OAAO,CAAA;CAClB;AAED,MAAM,WAAW,SAAS;IACtB,MAAM,EAAE,QAAQ,CAAA;CACnB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,GAAG,CAAC,KAAK,SAAS,SAAS,EAAE,KAAK,EAC9C,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,EACtB,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,EACnB,WAAW,EAAE,CAAC,GAAG,EAAE,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GACtE,IAAI,CASN"}

package/lib/run.js

@@ -22,13 +22,18 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.run = void 0;
+exports.run = run;
 const logger_1 = require("@subsquid/logger");
 const util_internal_1 = require("@subsquid/util-internal");
 const util_internal_prometheus_server_1 = require("@subsquid/util-internal-prometheus-server");
 const prom = __importStar(require("prom-client"));
 const metrics_1 = require("./metrics");
+const util_internal_data_source_1 = require("@subsquid/util-internal-data-source");
+const assert_1 = __importDefault(require("assert"));
 const util_1 = require("./util");
 const log = (0, logger_1.createLogger)('sqd:batch-processor');
 /**
@@ -48,11 +53,10 @@ const log = (0, logger_1.createLogger)('sqd:batch-processor');
 function run(src, db, dataHandler) {
     (0, util_internal_1.runProgram)(() => {
         return new Processor(src, db, dataHandler).run();
-    }, err => {
+    }, (err) => {
         log.fatal(err);
     });
 }
-exports.run = run;
 class Processor {
     constructor(src, db, handler) {
         this.src = src;
@@ -60,32 +64,73 @@ class Processor {
         this.handler = handler;
         this.metrics = new metrics_1.Metrics();
         this.hasStatusNews = false;
-        this.chainHeight = new util_internal_1.Throttler(() => this.src.getFinalizedHeight(), 30000);
+        this.chainHeight = new util_internal_1.Throttler(() => this.src.getHead()?.then((r) => r?.number ?? -1), 30000);
     }
     async run() {
-        let state = await this.db.connect();
-        if (state.height >= 0) {
-            log.info(`last processed final block was ${state.height}`);
+        let finalizedHead;
+        let head;
+        if (this.db.supportsHotBlocks) {
+            let state = await this.db.connect();
+            finalizedHead = state;
+            head = (0, util_internal_1.last)([state, ...state.top]);
         }
-        await this.assertWeAreOnTheSameChain(state);
-        await this.initMetrics(state);
-        for await (let blocks of this.src.getBlockStream(state.height + 1)) {
-            if (blocks.length > 0) {
-                state = await this.processBatch(state, blocks);
+        else {
+            finalizedHead = head = await this.db.connect();
+        }
+        if (head.height >= 0) {
+            log.info(`last processed block was ${head.height}`);
+            await this.assertWeAreOnTheSameChain(finalizedHead);
+        }
+        await this.initMetrics(head.height);
+        while (true) {
+            try {
+                let prevBlockNumber = head.height;
+                let prevBlockHash = head.height < 0 ? undefined : head.hash;
+                let stream = this.src.getStream({
+                    range: { from: prevBlockNumber + 1 },
+                    parentHash: prevBlockHash,
+                });
+                for await (let data of stream) {
+                    let finalizedHead = data.finalizedHead == null
+                        ? { height: -1, hash: '0x' }
+                        : {
+                            height: data.finalizedHead.number,
+                            hash: data.finalizedHead.hash,
+                        };
+                    head = await this.processBatch(head, finalizedHead, data.blocks);
+                }
+                break;
+            }
+            catch (e) {
+                if ((0, util_internal_data_source_1.isForkException)(e) && this.db.supportsHotBlocks) {
+                    let state = await this.db.getState();
+                    let forkBase = await computeForkBase(state, e.prevBlocks);
+                    if (forkBase == null) {
+                        // rollback all blocks
+                        head = { height: -1, hash: '0x' };
+                    }
+                    else {
+                        head = forkBase;
+                    }
+                    log.info(`navigating a fork on a common base ${(0, util_1.formatHead)(head)}`);
+                }
+                else {
+                    throw e;
+                }
             }
         }
         this.reportFinalStatus();
     }
     async assertWeAreOnTheSameChain(state) {
-        if (state.height < 0)
-            return;
-        let hash = await this.src.getBlockHash(state.height);
-        if (state.hash === hash)
-            return;
-        throw new Error(`already indexed block ${(0, util_1.formatHead)(state)} was not found on chain`);
+        // if (state.height < 0) return
+        // let hash = await this.src.getBlockHash(state.number)
+        // if (state.hash === hash) return
+        // throw new Error(
+        //     `already indexed block ${formatHead(state)} was not found on chain`
+        // )
     }
     async initMetrics(state) {
-        await this.updateProgressMetrics(await this.chainHeight.get(), state);
+        this.updateProgressMetrics(await this.chainHeight.get(), state);
         let port = process.env.PROCESSOR_PROMETHEUS_PORT || process.env.PROMETHEUS_PORT;
         if (port == null)
             return;
@@ -94,48 +139,57 @@ class Processor {
         let server = await (0, util_internal_prometheus_server_1.createPrometheusServer)(prom.register, port);
         log.info(`prometheus metrics are served on port ${server.port}`);
     }
-    updateProgressMetrics(chainHeight, state, time) {
+    updateProgressMetrics(chainHeight, indexerHeight, time) {
         this.metrics.setChainHeight(chainHeight);
-        this.metrics.setLastProcessedBlock(state.height);
+        this.metrics.setLastProcessedBlock(indexerHeight);
         let left;
         let processed;
-        if (this.src.getBlocksCountInRange) {
-            left = this.src.getBlocksCountInRange({
-                from: this.metrics.getLastProcessedBlock() + 1,
-                to: this.metrics.getChainHeight()
-            });
-            processed = this.src.getBlocksCountInRange({
-                from: 0,
-                to: this.metrics.getChainHeight()
-            }) - left;
-        }
-        else {
-            left = this.metrics.getChainHeight() - this.metrics.getLastProcessedBlock();
-            processed = this.metrics.getLastProcessedBlock();
-        }
+        left = this.metrics.getChainHeight() - this.metrics.getLastProcessedBlock();
+        processed = this.metrics.getLastProcessedBlock();
         this.metrics.updateProgress(processed, left, time);
     }
-    async processBatch(prevHead, blocks) {
+    async processBatch(prevHead, finalizedHead, blocks) {
         let chainHeight = await this.chainHeight.get();
+        let lastBlock = (0, util_internal_1.maybeLast)(blocks);
+        if (lastBlock == null)
+            return prevHead;
         let nextHead = {
-            hash: (0, util_internal_1.last)(blocks).header.hash,
-            height: (0, util_internal_1.last)(blocks).header.height
+            hash: lastBlock.header.hash,
+            height: lastBlock.header.number,
         };
         let isOnTop = nextHead.height >= chainHeight;
         let mappingStartTime = process.hrtime.bigint();
-        await this.db.transact({
-            prevHead,
-            nextHead,
-            isOnTop
-        }, store => {
-            return this.handler({
-                store,
-                blocks,
-                isHead: isOnTop
+        if (this.db.supportsHotBlocks) {
+            await this.db.transactHot2({
+                finalizedHead,
+                baseHead: prevHead,
+                newBlocks: blocks.map((b) => ({
+                    hash: b.header.hash,
+                    height: b.header.number,
+                })),
+            }, (store, start, end) => {
+                return this.handler({
+                    store,
+                    blocks: blocks.slice(start, end),
+                    isHead: isOnTop,
+                });
             });
-        });
+        }
+        else {
+            await this.db.transact({
+                prevHead,
+                nextHead,
+                isOnTop,
+            }, (store) => {
+                return this.handler({
+                    store,
+                    blocks,
+                    isHead: isOnTop,
+                });
+            });
+        }
         let mappingEndTime = process.hrtime.bigint();
-        this.updateProgressMetrics(chainHeight, nextHead, mappingEndTime);
+        this.updateProgressMetrics(chainHeight, nextHead.height, mappingEndTime);
         this.metrics.registerBatch(blocks.length, (0, util_1.getItemsCount)(blocks), mappingStartTime, mappingEndTime);
         this.reportStatus();
         return nextHead;
@@ -165,4 +219,33 @@ class Processor {
         }
     }
 }
+async function computeForkBase(state, forked, finalizedHead) {
+    (0, assert_1.default)(forked?.length);
+    let tail = forked.slice();
+    let commited = state.top;
+    if (commited.length > 0) {
+        for (let i = commited.length - 1; i >= 0; i--) {
+            let h = commited[i];
+            while (tail.length > 0 && (0, util_internal_1.last)(tail).number > h.height) {
+                tail.pop();
+            }
+            if (tail.length == 0)
+                return h;
+            let t = (0, util_internal_1.last)(tail);
+            if (t.number == h.height && t.hash == h.hash)
+                return h;
+        }
+    }
+    else {
+        if (forked[0].number > state.height)
+            return state;
+        let headOnChain = forked.find((b) => b.number == state.height);
+        if (headOnChain == null || headOnChain.hash !== state.hash) {
+            if (finalizedHead && finalizedHead.number >= state.height) {
+                throw new Error(`finalized block ${(0, util_1.formatHead)(state)} was not found on chain`);
+            }
+        }
+        return state;
+    }
+}
 //# sourceMappingURL=run.js.map

package/lib/run.js.map

@@ -1 +1 @@
-{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA6C;AAC7C,2DAAmE;AACnE,+FAAgF;AAChF,kDAAmC;AAGnC,uCAAiC;AACjC,iCAAgD;AAGhD,MAAM,GAAG,GAAG,IAAA,qBAAY,EAAC,qBAAqB,CAAC,CAAA;AAwB/C;;;;;;;;;;;;;GAaG;AACH,SAAgB,GAAG,CACf,GAAsB,EACtB,EAAmB,EACnB,WAAqE;IAErE,IAAA,0BAAU,EAAC,GAAG,EAAE;QACZ,OAAO,IAAI,SAAS,CAAC,GAAG,EAAE,EAAE,EAAE,WAAW,CAAC,CAAC,GAAG,EAAE,CAAA;IACpD,CAAC,EAAE,GAAG,CAAC,EAAE;QACL,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAClB,CAAC,CAAC,CAAA;AACN,CAAC;AAVD,kBAUC;AAGD,MAAM,SAAS;IAMX,YACY,GAAkB,EAClB,EAAe,EACf,OAAyD;QAFzD,QAAG,GAAH,GAAG,CAAe;QAClB,OAAE,GAAF,EAAE,CAAa;QACf,YAAO,GAAP,OAAO,CAAkD;QAR7D,YAAO,GAAG,IAAI,iBAAO,EAAE,CAAA;QAGvB,kBAAa,GAAG,KAAK,CAAA;QAOzB,IAAI,CAAC,WAAW,GAAG,IAAI,yBAAS,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,EAAE,KAAM,CAAC,CAAA;IACjF,CAAC;IAED,KAAK,CAAC,GAAG;QACL,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAA;QACnC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,kCAAkC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,MAAM,IAAI,CAAC,yBAAyB,CAAC,KAAK,CAAC,CAAA;QAC3C,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;QAE7B,IAAI,KAAK,EAAE,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YACjE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;YAClD,CAAC;QACL,CAAC;QAED,IAAI,CAAC,iBAAiB,EAAE,CAAA;IAC5B,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,KAAoB;QACxD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,OAAM;QAC5B,IAAI,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QACpD,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI;YAAE,OAAM;QAC/B,MAAM,IAAI,KAAK,CACX,yBAAyB,IAAA,iBAAU,EAAC,KAAK,CAAC,yBAAyB,CACtE,CAAA;IACL,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAAoB;QAC1C,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;QACrE,IAAI,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAA;QAC/E,IAAI,IAAI,IAAI,IAAI;YAAE,OAAM;QACxB,IAAI,CAAC,qBAAqB,EAAE,CAAA;QAC5B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACtB,IAAI,MAAM,GAAG,MAAM,IAAA,wDAAsB,EAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;QAC9D,GAAG,CAAC,IAAI,CAAC,yCAAyC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IACpE,CAAC;IAEO,qBAAqB,CAAC,WAAmB,EAAE,KAAoB,EAAE,IAAa;QAClF,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAChD,IAAI,IAAY,CAAA;QAChB,IAAI,SAAiB,CAAA;QACrB,IAAI,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC;YACjC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,CAAC;gBAClC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,GAAG,CAAC;gBAC9C,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE;aACpC,CAAC,CAAA;YACF,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,CAAC;gBACvC,IAAI,EAAE,CAAC;gBACP,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE;aACpC,CAAC,GAAG,IAAI,CAAA;QACb,CAAC;aAAM,CAAC;YACJ,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAA;YAC3E,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAA;QACpD,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;IACtD,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,QAAuB,EAAE,MAAW;QAC3D,IAAI,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAA;QAE9C,IAAI,QAAQ,GAAG;YACX,IAAI,EAAE,IAAA,oBAAI,EAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI;YAC9B,MAAM,EAAE,IAAA,oBAAI,EAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM;SACrC,CAAA;QAED,IAAI,OAAO,GAAG,QAAQ,CAAC,MAAM,IAAI,WAAW,CAAA;QAE5C,IAAI,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QAE9C,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC;YACnB,QAAQ;YACR,QAAQ;YACR,OAAO;SACV,EAAE,KAAK,CAAC,EAAE;YACP,OAAO,IAAI,CAAC,OAAO,CAAC;gBAChB,KAAK;gBACL,MAAM;gBACN,MAAM,EAAE,OAAO;aAClB,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;QAEF,IAAI,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QAE5C,IAAI,CAAC,qBAAqB,CAAC,WAAW,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAA;QACjE,IAAI,CAAC,OAAO,CAAC,aAAa,CACtB,MAAM,CAAC,MAAM,EACb,IAAA,oBAAa,EAAC,MAAM,CAAC,EACrB,gBAAgB,EAChB,cAAc,CACjB,CAAA;QAED,IAAI,CAAC,YAAY,EAAE,CAAA;QAEnB,OAAO,QAAQ,CAAA;IACnB,CAAC;IAEO,YAAY;QAChB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;YACtC,IAAI,CAAC,iBAAiB,GAAG,UAAU,CAAC,GAAG,EAAE;gBACrC,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAA;gBAClC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;oBACrB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;oBAC1B,IAAI,CAAC,YAAY,EAAE,CAAA;gBACvB,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,CAAA;QACZ,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;QAC7B,CAAC;IACL,CAAC;IAEO,iBAAiB;QACrB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACjC,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACxC,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;YAC1B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;QAC1C,CAAC;IACL,CAAC;CACJ"}
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6CA,kBAaC;AA1DD,6CAA6C;AAC7C,2DAA8E;AAC9E,+FAAgF;AAChF,kDAAmC;AAEnC,uCAAiC;AACjC,mFAAyF;AACzF,oDAA2B;AAC3B,iCAAgD;AAEhD,MAAM,GAAG,GAAG,IAAA,qBAAY,EAAC,qBAAqB,CAAC,CAAA;AAqB/C;;;;;;;;;;;;;GAaG;AACH,SAAgB,GAAG,CACf,GAAsB,EACtB,EAAmB,EACnB,WAAqE;IAErE,IAAA,0BAAU,EACN,GAAG,EAAE;QACD,OAAO,IAAI,SAAS,CAAC,GAAG,EAAE,EAAE,EAAE,WAAW,CAAC,CAAC,GAAG,EAAE,CAAA;IACpD,CAAC,EACD,CAAC,GAAG,EAAE,EAAE;QACJ,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAClB,CAAC,CACJ,CAAA;AACL,CAAC;AAED,MAAM,SAAS;IAMX,YACY,GAAkB,EAClB,EAAe,EACf,OAAyD;QAFzD,QAAG,GAAH,GAAG,CAAe;QAClB,OAAE,GAAF,EAAE,CAAa;QACf,YAAO,GAAP,OAAO,CAAkD;QAR7D,YAAO,GAAG,IAAI,iBAAO,EAAE,CAAA;QAGvB,kBAAa,GAAG,KAAK,CAAA;QAOzB,IAAI,CAAC,WAAW,GAAG,IAAI,yBAAS,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,KAAM,CAAC,CAAA;IACpG,CAAC;IAED,KAAK,CAAC,GAAG;QACL,IAAI,aAA4B,CAAA;QAChC,IAAI,IAAmB,CAAA;QACvB,IAAI,IAAI,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAA;YACnC,aAAa,GAAG,KAAK,CAAA;YACrB,IAAI,GAAG,IAAA,oBAAI,EAAC,CAAC,KAAK,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;QACtC,CAAC;aAAM,CAAC;YACJ,aAAa,GAAG,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAA;QAClD,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;YACnD,MAAM,IAAI,CAAC,yBAAyB,CAAC,aAAa,CAAC,CAAA;QACvD,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAEnC,OAAO,IAAI,EAAE,CAAC;YACV,IAAI,CAAC;gBACD,IAAI,eAAe,GAAG,IAAI,CAAC,MAAM,CAAA;gBACjC,IAAI,aAAa,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAA;gBAE3D,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;oBAC5B,KAAK,EAAE,EAAC,IAAI,EAAE,eAAe,GAAG,CAAC,EAAC;oBAClC,UAAU,EAAE,aAAa;iBAC5B,CAAC,CAAA;gBAEF,IAAI,KAAK,EAAE,IAAI,IAAI,IAAI,MAAM,EAAE,CAAC;oBAC5B,IAAI,aAAa,GACb,IAAI,CAAC,aAAa,IAAI,IAAI;wBACtB,CAAC,CAAC,EAAC,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAC;wBAC1B,CAAC,CAAC;4BACI,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;4BACjC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI;yBAChC,CAAA;oBAEX,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;gBACpE,CAAC;gBAED,MAAK;YACT,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,IAAA,2CAAe,EAAC,CAAC,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC;oBAClD,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAA;oBACpC,IAAI,QAAQ,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,CAAC,CAAC,UAAU,CAAC,CAAA;oBACzD,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;wBACnB,sBAAsB;wBACtB,IAAI,GAAG,EAAC,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAC,CAAA;oBACnC,CAAC;yBAAM,CAAC;wBACJ,IAAI,GAAG,QAAQ,CAAA;oBACnB,CAAC;oBACD,GAAG,CAAC,IAAI,CAAC,sCAAsC,IAAA,iBAAU,EAAC,IAAI,CAAC,EAAE,CAAC,CAAA;gBACtE,CAAC;qBAAM,CAAC;oBACJ,MAAM,CAAC,CAAA;gBACX,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,CAAC,iBAAiB,EAAE,CAAA;IAC5B,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,KAAoB;QACxD,+BAA+B;QAC/B,uDAAuD;QACvD,kCAAkC;QAClC,mBAAmB;QACnB,0EAA0E;QAC1E,IAAI;IACR,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAAa;QACnC,IAAI,CAAC,qBAAqB,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;QAC/D,IAAI,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAA;QAC/E,IAAI,IAAI,IAAI,IAAI;YAAE,OAAM;QACxB,IAAI,CAAC,qBAAqB,EAAE,CAAA;QAC5B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACtB,IAAI,MAAM,GAAG,MAAM,IAAA,wDAAsB,EAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;QAC9D,GAAG,CAAC,IAAI,CAAC,yCAAyC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IACpE,CAAC;IAEO,qBAAqB,CAAC,WAAmB,EAAE,aAAqB,EAAE,IAAa;QACnF,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,aAAa,CAAC,CAAA;QACjD,IAAI,IAAY,CAAA;QAChB,IAAI,SAAiB,CAAA;QACrB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAA;QAC3E,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAA;QAChD,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;IACtD,CAAC;IAEO,KAAK,CAAC,YAAY,CACtB,QAAuB,EACvB,aAA4B,EAC5B,MAAW;QAEX,IAAI,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAA;QAE9C,IAAI,SAAS,GAAG,IAAA,yBAAS,EAAC,MAAM,CAAC,CAAA;QACjC,IAAI,SAAS,IAAI,IAAI;YAAE,OAAO,QAAQ,CAAA;QAEtC,IAAI,QAAQ,GAAG;YACX,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,IAAI;YAC3B,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,MAAM;SAClC,CAAA;QAED,IAAI,OAAO,GAAG,QAAQ,CAAC,MAAM,IAAI,WAAW,CAAA;QAE5C,IAAI,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QAE9C,IAAI,IAAI,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC;YAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CACtB;gBACI,aAAa;gBACb,QAAQ,EAAE,QAAQ;gBAClB,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC1B,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI;oBACnB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM;iBAC1B,CAAC,CAAC;aACN,EACD,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;gBAClB,OAAO,IAAI,CAAC,OAAO,CAAC;oBAChB,KAAK;oBACL,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;oBAChC,MAAM,EAAE,OAAO;iBAClB,CAAC,CAAA;YACN,CAAC,CACJ,CAAA;QACL,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAClB;gBACI,QAAQ;gBACR,QAAQ;gBACR,OAAO;aACV,EACD,CAAC,KAAK,EAAE,EAAE;gBACN,OAAO,IAAI,CAAC,OAAO,CAAC;oBAChB,KAAK;oBACL,MAAM;oBACN,MAAM,EAAE,OAAO;iBAClB,CAAC,CAAA;YACN,CAAC,CACJ,CAAA;QACL,CAAC;QAED,IAAI,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QAE5C,IAAI,CAAC,qBAAqB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;QACxE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,IAAA,oBAAa,EAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAA;QAElG,IAAI,CAAC,YAAY,EAAE,CAAA;QAEnB,OAAO,QAAQ,CAAA;IACnB,CAAC;IAEO,YAAY;QAChB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;YACtC,IAAI,CAAC,iBAAiB,GAAG,UAAU,CAAC,GAAG,EAAE;gBACrC,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAA;gBAClC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;oBACrB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;oBAC1B,IAAI,CAAC,YAAY,EAAE,CAAA;gBACvB,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,CAAA;QACZ,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;QAC7B,CAAC;IACL,CAAC;IAEO,iBAAiB;QACrB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACjC,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACxC,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;YAC1B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;QAC1C,CAAC;IACL,CAAC;CACJ;AAED,KAAK,UAAU,eAAe,CAC1B,KAAuB,EACvB,MAAkB,EAClB,aAAwB;IAExB,IAAA,gBAAM,EAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACtB,IAAI,IAAI,GAAG,MAAM,CAAC,KAAK,EAAE,CAAA;IAEzB,IAAI,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAA;IACxB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;YAEnB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAA,oBAAI,EAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;gBACrD,IAAI,CAAC,GAAG,EAAE,CAAA;YACd,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC;gBAAE,OAAO,CAAC,CAAA;YAE9B,IAAI,CAAC,GAAG,IAAA,oBAAI,EAAC,IAAI,CAAC,CAAA;YAClB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI;gBAAE,OAAO,CAAC,CAAA;QAC1D,CAAC;IACL,CAAC;SAAM,CAAC;QACJ,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;YAAE,OAAO,KAAK,CAAA;QAEjD,IAAI,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAA;QAC9D,IAAI,WAAW,IAAI,IAAI,IAAI,WAAW,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;YACzD,IAAI,aAAa,IAAI,aAAa,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxD,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAA,iBAAU,EAAC,KAAK,CAAC,yBAAyB,CAAC,CAAA;YAClF,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAA;IAChB,CAAC;AACL,CAAC"}

package/lib/util.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.shortHash = exports.formatHead = exports.getItemsCount = exports.timeInterval = void 0;
+exports.timeInterval = timeInterval;
+exports.getItemsCount = getItemsCount;
+exports.formatHead = formatHead;
+exports.shortHash = shortHash;
 function timeInterval(seconds) {
     if (seconds < 60) {
         return Math.round(seconds) + 's';
@@ -13,7 +16,6 @@ function timeInterval(seconds) {
     minutes = minutes - hours * 60;
     return hours + 'h ' + minutes + 'm';
 }
-exports.timeInterval = timeInterval;
 function getItemsCount(blocks) {
     let count = 0;
     for (let block of blocks) {
@@ -26,11 +28,9 @@ function getItemsCount(blocks) {
     }
     return count;
 }
-exports.getItemsCount = getItemsCount;
 function formatHead(head) {
     return `${head.height}#${shortHash(head.hash)}`;
 }
-exports.formatHead = formatHead;
 function shortHash(hash) {
     if (hash.startsWith('0x')) {
         return hash.slice(2, 7);
@@ -39,5 +39,4 @@ function shortHash(hash) {
         return hash.slice(0, 5);
     }
 }
-exports.shortHash = shortHash;
 //# sourceMappingURL=util.js.map

package/lib/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;AAGA,SAAgB,YAAY,CAAC,OAAe;IACxC,IAAI,OAAO,GAAG,EAAE,EAAE,CAAC;QACf,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,GAAG,CAAA;IACpC,CAAC;IACD,IAAI,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,GAAC,EAAE,CAAC,CAAA;IACnC,IAAI,OAAO,GAAG,EAAE,EAAE,CAAC;QACf,OAAQ,OAAO,GAAC,GAAG,CAAA;IACvB,CAAC;IACD,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAA;IACpC,OAAO,GAAG,OAAO,GAAG,KAAK,GAAG,EAAE,CAAA;IAC9B,OAAO,KAAK,GAAG,IAAI,GAAG,OAAO,GAAG,GAAG,CAAA;AACvC,CAAC;AAXD,oCAWC;AAGD,SAAgB,aAAa,CAAC,MAAa;IACvC,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACvB,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;YACpB,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAA;YACpB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,KAAK,IAAI,GAAG,CAAC,MAAM,CAAA;YACvB,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAA;AAChB,CAAC;AAXD,sCAWC;AAGD,SAAgB,UAAU,CAAC,IAAmB;IAC1C,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAA;AACnD,CAAC;AAFD,gCAEC;AAGD,SAAgB,SAAS,CAAC,IAAY;IAClC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC3B,CAAC;SAAM,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC3B,CAAC;AACL,CAAC;AAND,8BAMC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;AAGA,oCAWC;AAGD,sCAWC;AAGD,gCAEC;AAGD,8BAMC;AAvCD,SAAgB,YAAY,CAAC,OAAe;IACxC,IAAI,OAAO,GAAG,EAAE,EAAE,CAAC;QACf,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,GAAG,CAAA;IACpC,CAAC;IACD,IAAI,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,GAAC,EAAE,CAAC,CAAA;IACnC,IAAI,OAAO,GAAG,EAAE,EAAE,CAAC;QACf,OAAQ,OAAO,GAAC,GAAG,CAAA;IACvB,CAAC;IACD,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAA;IACpC,OAAO,GAAG,OAAO,GAAG,KAAK,GAAG,EAAE,CAAA;IAC9B,OAAO,KAAK,GAAG,IAAI,GAAG,OAAO,GAAG,GAAG,CAAA;AACvC,CAAC;AAGD,SAAgB,aAAa,CAAC,MAAa;IACvC,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACvB,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;YACpB,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAA;YACpB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,KAAK,IAAI,GAAG,CAAC,MAAM,CAAA;YACvB,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAA;AAChB,CAAC;AAGD,SAAgB,UAAU,CAAC,IAAmB;IAC1C,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAA;AACnD,CAAC;AAGD,SAAgB,SAAS,CAAC,IAAY;IAClC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC3B,CAAC;SAAM,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC3B,CAAC;AACL,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@subsquid/batch-processor",
-  "version": "0.0.0",
+  "version": "1.0.0-portal-api.d887ad",
   "description": "ETL processor",
   "license": "GPL-3.0-or-later",
   "repository": "git@github.com:subsquid/squid.git",
@@ -14,16 +14,17 @@
   ],
   "main": "lib/index.js",
   "dependencies": {
-    "@subsquid/logger": "^1.3.3",
-    "@subsquid/util-internal": "^3.2.0",
+    "@subsquid/logger": "1.5.0-portal-api.d887ad",
+    "@subsquid/util-internal": "3.3.0-portal-api.d887ad",
     "@subsquid/util-internal-counters": "^1.3.2",
     "@subsquid/util-internal-prometheus-server": "^1.3.0",
     "@subsquid/util-internal-range": "^0.3.0",
-    "prom-client": "^14.2.0"
+    "prom-client": "^14.2.0",
+    "@subsquid/util-internal-data-source": "^0.0.0"
   },
   "devDependencies": {
     "@types/node": "^18.18.14",
-    "typescript": "~5.3.2"
+    "typescript": "5.5.4"
   },
   "scripts": {
     "build": "rm -rf lib && tsc"

package/src/database.ts

@@ -14,11 +14,15 @@ export interface FinalTxInfo {
 
 export interface FinalDatabase<S> {
     supportsHotBlocks?: false
-    connect(): Promise<HashAndHeight>
+    connect(): Promise<FinalDatabaseState>
+    getState?(): Promise<FinalDatabaseState>
     transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>
 }
 
 
+export interface FinalDatabaseState extends HashAndHeight {}
+
+
 export interface HotTxInfo {
     finalizedHead: HashAndHeight
     baseHead: HashAndHeight
@@ -29,13 +33,10 @@ export interface HotTxInfo {
 export interface HotDatabase<S> {
     supportsHotBlocks: true
     connect(): Promise<HotDatabaseState>
+    getState(): Promise<HotDatabaseState>
     transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>
-    /**
-     * @deprecated
-     */
-    transactHot(info: HotTxInfo, cb: (store: S, block: HashAndHeight) => Promise<void>): Promise<void>
 
-    transactHot2?(
+    transactHot2(
         info: HotTxInfo,
         cb: (store: S, blockSliceStart: number, blockSliceEnd: number) => Promise<void>
     ): Promise<void>
@@ -50,4 +51,4 @@ export interface HotDatabaseState extends HashAndHeight {
 export interface HashAndHeight {
     height: number
     hash: string
-}
+}

package/src/index.ts

@@ -1,3 +1,2 @@
 export * from './database'
-export * from './datasource'
 export * from './run'

package/src/run.ts

@@ -1,16 +1,15 @@
 import {createLogger} from '@subsquid/logger'
-import {last, runProgram, Throttler} from '@subsquid/util-internal'
+import {last, maybeLast, runProgram, Throttler} from '@subsquid/util-internal'
 import {createPrometheusServer} from '@subsquid/util-internal-prometheus-server'
 import * as prom from 'prom-client'
-import {Database, HashAndHeight} from './database'
-import {DataSource} from './datasource'
+import {HashAndHeight, Database, HotDatabaseState} from './database'
 import {Metrics} from './metrics'
+import {DataSource, isForkException, BlockRef} from '@subsquid/util-internal-data-source'
+import assert from 'assert'
 import {formatHead, getItemsCount} from './util'
 
-
 const log = createLogger('sqd:batch-processor')
 
-
 export interface DataHandlerContext<Block, Store> {
     /**
      * Storage interface provided by the database
@@ -26,12 +25,10 @@ export interface DataHandlerContext<Block, Store> {
     isHead: boolean
 }
 
-
-interface BlockBase {
-    header: HashAndHeight
+export interface BlockBase {
+    header: BlockRef
 }
 
-
 /**
  * Run data processing.
  *
@@ -51,14 +48,16 @@ export function run<Block extends BlockBase, Store>(
     db: Database<Store>,
     dataHandler: (ctx: DataHandlerContext<Block, Store>) => Promise<void>
 ): void {
-    runProgram(() => {
-        return new Processor(src, db, dataHandler).run()
-    }, err => {
-        log.fatal(err)
-    })
+    runProgram(
+        () => {
+            return new Processor(src, db, dataHandler).run()
+        },
+        (err) => {
+            log.fatal(err)
+        }
+    )
 }
 
-
 class Processor<B extends BlockBase, S> {
     private metrics = new Metrics()
     private chainHeight: Throttler<number>
@@ -70,21 +69,64 @@ class Processor<B extends BlockBase, S> {
         private db: Database<S>,
         private handler: (ctx: DataHandlerContext<B, S>) => Promise<void>
     ) {
-        this.chainHeight = new Throttler(() => this.src.getFinalizedHeight(), 30_000)
+        this.chainHeight = new Throttler(() => this.src.getHead()?.then((r) => r?.number ?? -1), 30_000)
     }
 
     async run(): Promise<void> {
-        let state = await this.db.connect()
-        if (state.height >= 0) {
-            log.info(`last processed final block was ${state.height}`)
+        let finalizedHead: HashAndHeight
+        let head: HashAndHeight
+        if (this.db.supportsHotBlocks) {
+            let state = await this.db.connect()
+            finalizedHead = state
+            head = last([state, ...state.top])
+        } else {
+            finalizedHead = head = await this.db.connect()
+        }
+
+        if (head.height >= 0) {
+            log.info(`last processed block was ${head.height}`)
+            await this.assertWeAreOnTheSameChain(finalizedHead)
         }
 
-        await this.assertWeAreOnTheSameChain(state)
-        await this.initMetrics(state)
+        await this.initMetrics(head.height)
 
-        for await (let blocks of this.src.getBlockStream(state.height + 1)) {
-            if (blocks.length > 0) {
-                state = await this.processBatch(state, blocks)
+        while (true) {
+            try {
+                let prevBlockNumber = head.height
+                let prevBlockHash = head.height < 0 ? undefined : head.hash
+
+                let stream = this.src.getStream({
+                    range: {from: prevBlockNumber + 1},
+                    parentHash: prevBlockHash,
+                })
+
+                for await (let data of stream) {
+                    let finalizedHead: HashAndHeight =
+                        data.finalizedHead == null
+                            ? {height: -1, hash: '0x'}
+                            : {
+                                  height: data.finalizedHead.number,
+                                  hash: data.finalizedHead.hash,
+                              }
+
+                    head = await this.processBatch(head, finalizedHead, data.blocks)
+                }
+
+                break
+            } catch (e) {
+                if (isForkException(e) && this.db.supportsHotBlocks) {
+                    let state = await this.db.getState()
+                    let forkBase = await computeForkBase(state, e.prevBlocks)
+                    if (forkBase == null) {
+                        // rollback all blocks
+                        head = {height: -1, hash: '0x'}
+                    } else {
+                        head = forkBase
+                    }
+                    log.info(`navigating a fork on a common base ${formatHead(head)}`)
+                } else {
+                    throw e
+                }
             }
         }
 
@@ -92,16 +134,16 @@ class Processor<B extends BlockBase, S> {
     }
 
     private async assertWeAreOnTheSameChain(state: HashAndHeight): Promise<void> {
-        if (state.height < 0) return
-        let hash = await this.src.getBlockHash(state.height)
-        if (state.hash === hash) return
-        throw new Error(
-            `already indexed block ${formatHead(state)} was not found on chain`
-        )
+        // if (state.height < 0) return
+        // let hash = await this.src.getBlockHash(state.number)
+        // if (state.hash === hash) return
+        // throw new Error(
+        //     `already indexed block ${formatHead(state)} was not found on chain`
+        // )
     }
 
-    private async initMetrics(state: HashAndHeight): Promise<void> {
-        await this.updateProgressMetrics(await this.chainHeight.get(), state)
+    private async initMetrics(state: number): Promise<void> {
+        this.updateProgressMetrics(await this.chainHeight.get(), state)
         let port = process.env.PROCESSOR_PROMETHEUS_PORT || process.env.PROMETHEUS_PORT
         if (port == null) return
         prom.collectDefaultMetrics()
@@ -110,60 +152,74 @@ class Processor<B extends BlockBase, S> {
         log.info(`prometheus metrics are served on port ${server.port}`)
     }
 
-    private updateProgressMetrics(chainHeight: number, state: HashAndHeight, time?: bigint): void {
+    private updateProgressMetrics(chainHeight: number, indexerHeight: number, time?: bigint): void {
         this.metrics.setChainHeight(chainHeight)
-        this.metrics.setLastProcessedBlock(state.height)
+        this.metrics.setLastProcessedBlock(indexerHeight)
         let left: number
         let processed: number
-        if (this.src.getBlocksCountInRange) {
-            left = this.src.getBlocksCountInRange({
-                from: this.metrics.getLastProcessedBlock() + 1,
-                to: this.metrics.getChainHeight()
-            })
-            processed = this.src.getBlocksCountInRange({
-                from: 0,
-                to: this.metrics.getChainHeight()
-            }) - left
-        } else {
-            left = this.metrics.getChainHeight() - this.metrics.getLastProcessedBlock()
-            processed = this.metrics.getLastProcessedBlock()
-        }
+        left = this.metrics.getChainHeight() - this.metrics.getLastProcessedBlock()
+        processed = this.metrics.getLastProcessedBlock()
         this.metrics.updateProgress(processed, left, time)
     }
 
-    private async processBatch(prevHead: HashAndHeight, blocks: B[]): Promise<HashAndHeight> {
+    private async processBatch(
+        prevHead: HashAndHeight,
+        finalizedHead: HashAndHeight,
+        blocks: B[]
+    ): Promise<HashAndHeight> {
         let chainHeight = await this.chainHeight.get()
 
+        let lastBlock = maybeLast(blocks)
+        if (lastBlock == null) return prevHead
+
         let nextHead = {
-            hash: last(blocks).header.hash,
-            height: last(blocks).header.height
+            hash: lastBlock.header.hash,
+            height: lastBlock.header.number,
         }
 
         let isOnTop = nextHead.height >= chainHeight
 
         let mappingStartTime = process.hrtime.bigint()
 
-        await this.db.transact({
-            prevHead,
-            nextHead,
-            isOnTop
-        }, store => {
-            return this.handler({
-                store,
-                blocks,
-                isHead: isOnTop
-            })
-        })
+        if (this.db.supportsHotBlocks) {
+            await this.db.transactHot2(
+                {
+                    finalizedHead,
+                    baseHead: prevHead,
+                    newBlocks: blocks.map((b) => ({
+                        hash: b.header.hash,
+                        height: b.header.number,
+                    })),
+                },
+                (store, start, end) => {
+                    return this.handler({
+                        store,
+                        blocks: blocks.slice(start, end),
+                        isHead: isOnTop,
+                    })
+                }
+            )
+        } else {
+            await this.db.transact(
+                {
+                    prevHead,
+                    nextHead,
+                    isOnTop,
+                },
+                (store) => {
+                    return this.handler({
+                        store,
+                        blocks,
+                        isHead: isOnTop,
+                    })
+                }
+            )
+        }
 
         let mappingEndTime = process.hrtime.bigint()
 
-        this.updateProgressMetrics(chainHeight, nextHead, mappingEndTime)
-        this.metrics.registerBatch(
-            blocks.length,
-            getItemsCount(blocks),
-            mappingStartTime,
-            mappingEndTime
-        )
+        this.updateProgressMetrics(chainHeight, nextHead.height, mappingEndTime)
+        this.metrics.registerBatch(blocks.length, getItemsCount(blocks), mappingStartTime, mappingEndTime)
 
         this.reportStatus()
 
@@ -195,3 +251,39 @@ class Processor<B extends BlockBase, S> {
         }
     }
 }
+
+async function computeForkBase(
+    state: HotDatabaseState,
+    forked: BlockRef[],
+    finalizedHead?: BlockRef
+): Promise<HashAndHeight | undefined> {
+    assert(forked?.length)
+    let tail = forked.slice()
+
+    let commited = state.top
+    if (commited.length > 0) {
+        for (let i = commited.length - 1; i >= 0; i--) {
+            let h = commited[i]
+
+            while (tail.length > 0 && last(tail).number > h.height) {
+                tail.pop()
+            }
+
+            if (tail.length == 0) return h
+
+            let t = last(tail)
+            if (t.number == h.height && t.hash == h.hash) return h
+        }
+    } else {
+        if (forked[0].number > state.height) return state
+
+        let headOnChain = forked.find((b) => b.number == state.height)
+        if (headOnChain == null || headOnChain.hash !== state.hash) {
+            if (finalizedHead && finalizedHead.number >= state.height) {
+                throw new Error(`finalized block ${formatHead(state)} was not found on chain`)
+            }
+        }
+
+        return state
+    }
+}

package/lib/datasource.d.ts

@@ -1,8 +0,0 @@
-import type { FiniteRange } from '@subsquid/util-internal-range';
-export interface DataSource<B> {
-    getFinalizedHeight(): Promise<number>;
-    getBlockHash(height: number): Promise<string | undefined>;
-    getBlockStream(fromBlock?: number): AsyncIterable<B[]>;
-    getBlocksCountInRange?(range: FiniteRange): number;
-}
-//# sourceMappingURL=datasource.d.ts.map

package/lib/datasource.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"datasource.d.ts","sourceRoot":"","sources":["../src/datasource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,+BAA+B,CAAA;AAG9D,MAAM,WAAW,UAAU,CAAC,CAAC;IACzB,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;IACrC,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAA;IACzD,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,CAAC,EAAE,CAAC,CAAA;IACtD,qBAAqB,CAAC,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,CAAA;CACrD"}

package/lib/datasource.js

@@ -1,3 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-//# sourceMappingURL=datasource.js.map

package/lib/datasource.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"datasource.js","sourceRoot":"","sources":["../src/datasource.ts"],"names":[],"mappings":""}

package/src/datasource.ts

@@ -1,9 +0,0 @@
-import type {FiniteRange} from '@subsquid/util-internal-range'
-
-
-export interface DataSource<B> {
-    getFinalizedHeight(): Promise<number>
-    getBlockHash(height: number): Promise<string | undefined>
-    getBlockStream(fromBlock?: number): AsyncIterable<B[]>
-    getBlocksCountInRange?(range: FiniteRange): number
-}