@subsquid/batch-processor 0.0.0

package/README.md

@@ -0,0 +1,4 @@
+# @subsquid/batch-processor
+
+Batch processor ingests block data from the data source
+and executes user defined data handler against received sequential batches of blocks.

package/lib/database.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Database is responsible for providing a persistent storage for data handlers
+ * and keeping the processor progress and status.
+ */
+export type Database<S> = FinalDatabase<S> | HotDatabase<S>;
+export interface FinalTxInfo {
+    prevHead: HashAndHeight;
+    nextHead: HashAndHeight;
+    isOnTop: boolean;
+}
+export interface FinalDatabase<S> {
+    supportsHotBlocks?: false;
+    connect(): Promise<HashAndHeight>;
+    transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>;
+}
+export interface HotTxInfo {
+    finalizedHead: HashAndHeight;
+    baseHead: HashAndHeight;
+    newBlocks: HashAndHeight[];
+}
+export interface HotDatabase<S> {
+    supportsHotBlocks: true;
+    connect(): Promise<HotDatabaseState>;
+    transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>;
+    /**
+     * @deprecated
+     */
+    transactHot(info: HotTxInfo, cb: (store: S, block: HashAndHeight) => Promise<void>): Promise<void>;
+    transactHot2?(info: HotTxInfo, cb: (store: S, blockSliceStart: number, blockSliceEnd: number) => Promise<void>): Promise<void>;
+}
+export interface HotDatabaseState extends HashAndHeight {
+    top: HashAndHeight[];
+}
+export interface HashAndHeight {
+    height: number;
+    hash: string;
+}
+//# sourceMappingURL=database.d.ts.map

package/lib/database.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../src/database.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;AAG3D,MAAM,WAAW,WAAW;IACxB,QAAQ,EAAE,aAAa,CAAA;IACvB,QAAQ,EAAE,aAAa,CAAA;IACvB,OAAO,EAAE,OAAO,CAAA;CACnB;AAGD,MAAM,WAAW,aAAa,CAAC,CAAC;IAC5B,iBAAiB,CAAC,EAAE,KAAK,CAAA;IACzB,OAAO,IAAI,OAAO,CAAC,aAAa,CAAC,CAAA;IACjC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9E;AAGD,MAAM,WAAW,SAAS;IACtB,aAAa,EAAE,aAAa,CAAA;IAC5B,QAAQ,EAAE,aAAa,CAAA;IACvB,SAAS,EAAE,aAAa,EAAE,CAAA;CAC7B;AAGD,MAAM,WAAW,WAAW,CAAC,CAAC;IAC1B,iBAAiB,EAAE,IAAI,CAAA;IACvB,OAAO,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAA;IACpC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC3E;;OAEG;IACH,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAElG,YAAY,CAAC,CACT,IAAI,EAAE,SAAS,EACf,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,eAAe,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAChF,OAAO,CAAC,IAAI,CAAC,CAAA;CACnB;AAGD,MAAM,WAAW,gBAAiB,SAAQ,aAAa;IACnD,GAAG,EAAE,aAAa,EAAE,CAAA;CACvB;AAGD,MAAM,WAAW,aAAa;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACf"}

package/lib/database.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=database.js.map

package/lib/database.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"database.js","sourceRoot":"","sources":["../src/database.ts"],"names":[],"mappings":""}

package/lib/datasource.d.ts

@@ -0,0 +1,8 @@
+import type { FiniteRange } from '@subsquid/util-internal-range';
+export interface DataSource<B> {
+    getFinalizedHeight(): Promise<number>;
+    getBlockHash(height: number): Promise<string | undefined>;
+    getBlockStream(fromBlock?: number): AsyncIterable<B[]>;
+    getBlocksCountInRange?(range: FiniteRange): number;
+}
+//# sourceMappingURL=datasource.d.ts.map

package/lib/datasource.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"datasource.d.ts","sourceRoot":"","sources":["../src/datasource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,+BAA+B,CAAA;AAG9D,MAAM,WAAW,UAAU,CAAC,CAAC;IACzB,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;IACrC,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAA;IACzD,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,CAAC,EAAE,CAAC,CAAA;IACtD,qBAAqB,CAAC,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,CAAA;CACrD"}

package/lib/datasource.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=datasource.js.map

package/lib/datasource.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"datasource.js","sourceRoot":"","sources":["../src/datasource.ts"],"names":[],"mappings":""}

package/lib/index.d.ts

@@ -0,0 +1,4 @@
+export * from './database';
+export * from './datasource';
+export * from './run';
+//# sourceMappingURL=index.d.ts.map

package/lib/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAA;AAC1B,cAAc,cAAc,CAAA;AAC5B,cAAc,OAAO,CAAA"}

package/lib/index.js

@@ -0,0 +1,20 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./database"), exports);
+__exportStar(require("./datasource"), exports);
+__exportStar(require("./run"), exports);
+//# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6CAA0B;AAC1B,+CAA4B;AAC5B,wCAAqB"}

package/lib/metrics.d.ts

@@ -0,0 +1,21 @@
+export declare class Metrics {
+    private chainHeight;
+    private lastBlock;
+    private mappingSpeed;
+    private mappingItemSpeed;
+    private blockProgress;
+    setChainHeight(height: number): void;
+    setLastProcessedBlock(height: number): void;
+    updateProgress(processed: number, left: number, time?: bigint): void;
+    registerBatch(batchSize: number, batchItemSize: number, batchMappingStartTime: bigint, batchMappingEndTime: bigint): void;
+    getChainHeight(): number;
+    getLastProcessedBlock(): number;
+    getSyncSpeed(): number;
+    getSyncEtaSeconds(): number;
+    getSyncRatio(): number;
+    getMappingSpeed(): number;
+    getMappingItemSpeed(): number;
+    getStatusLine(): string;
+    install(): void;
+}
+//# sourceMappingURL=metrics.d.ts.map

package/lib/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAKA,qBAAa,OAAO;IAChB,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,gBAAgB,CAA6B;IACrD,OAAO,CAAC,aAAa,CAAkD;IAEvE,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAIpC,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAK3C,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI;IAKpE,aAAa,CACT,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,qBAAqB,EAAE,MAAM,EAC7B,mBAAmB,EAAE,MAAM,GAC5B,IAAI;IAKP,cAAc,IAAI,MAAM;IAIxB,qBAAqB,IAAI,MAAM;IAI/B,YAAY,IAAI,MAAM;IAItB,iBAAiB,IAAI,MAAM;IAI3B,YAAY,IAAI,MAAM;IAItB,eAAe,IAAI,MAAM;IAIzB,mBAAmB,IAAI,MAAM;IAI7B,aAAa,IAAI,MAAM;IAQvB,OAAO,IAAI,IAAI;CA+BlB"}

package/lib/metrics.js

@@ -0,0 +1,92 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Metrics = void 0;
+const util_internal_counters_1 = require("@subsquid/util-internal-counters");
+const prom_client_1 = require("prom-client");
+const util_1 = require("./util");
+class Metrics {
+    constructor() {
+        this.chainHeight = -1;
+        this.lastBlock = -1;
+        this.mappingSpeed = new util_internal_counters_1.Speed({ windowSize: 5 });
+        this.mappingItemSpeed = new util_internal_counters_1.Speed({ windowSize: 5 });
+        this.blockProgress = new util_internal_counters_1.Progress({ initialValue: 0, windowSize: 20 });
+    }
+    setChainHeight(height) {
+        this.chainHeight = Math.max(height, this.lastBlock);
+    }
+    setLastProcessedBlock(height) {
+        this.lastBlock = height;
+        this.chainHeight = Math.max(this.chainHeight, this.lastBlock);
+    }
+    updateProgress(processed, left, time) {
+        this.blockProgress.setTargetValue(processed + left);
+        this.blockProgress.setCurrentValue(processed, time);
+    }
+    registerBatch(batchSize, batchItemSize, batchMappingStartTime, batchMappingEndTime) {
+        this.mappingSpeed.push(batchSize, batchMappingStartTime, batchMappingEndTime);
+        this.mappingItemSpeed.push(batchItemSize || 1, batchMappingStartTime, batchMappingEndTime);
+    }
+    getChainHeight() {
+        return this.chainHeight;
+    }
+    getLastProcessedBlock() {
+        return this.lastBlock;
+    }
+    getSyncSpeed() {
+        return this.blockProgress.speed();
+    }
+    getSyncEtaSeconds() {
+        return this.blockProgress.eta();
+    }
+    getSyncRatio() {
+        return this.blockProgress.ratio();
+    }
+    getMappingSpeed() {
+        return this.mappingSpeed.speed();
+    }
+    getMappingItemSpeed() {
+        return this.mappingItemSpeed.speed();
+    }
+    getStatusLine() {
+        return `${this.lastBlock} / ${this.chainHeight}, ` +
+            `rate: ${Math.round(this.getSyncSpeed())} blocks/sec, ` +
+            `mapping: ${Math.round(this.getMappingSpeed())} blocks/sec, ` +
+            `${Math.round(this.getMappingItemSpeed())} items/sec, ` +
+            `eta: ${(0, util_1.timeInterval)(this.getSyncEtaSeconds())}`;
+    }
+    install() {
+        new prom_client_1.Gauge({
+            name: 'sqd_processor_chain_height',
+            help: 'Chain height of the data source',
+            collect: collect(() => this.getChainHeight())
+        });
+        new prom_client_1.Gauge({
+            name: 'sqd_processor_last_block',
+            help: 'Last processed block',
+            collect: collect(() => this.getLastProcessedBlock())
+        });
+        new prom_client_1.Gauge({
+            name: 'sqd_processor_mapping_blocks_per_second',
+            help: 'Mapping performance',
+            collect: collect(() => this.getMappingSpeed())
+        });
+        new prom_client_1.Gauge({
+            name: 'sqd_processor_sync_eta_seconds',
+            help: 'Estimated time until all required blocks will be processed or until the chain height will be reached',
+            collect: collect(() => this.getSyncEtaSeconds())
+        });
+        new prom_client_1.Gauge({
+            name: 'sqd_processor_sync_ratio',
+            help: 'Percentage of processed blocks',
+            collect: collect(() => this.getSyncRatio())
+        });
+    }
+}
+exports.Metrics = Metrics;
+function collect(fn) {
+    return function () {
+        this.set(fn());
+    };
+}
+//# sourceMappingURL=metrics.js.map

package/lib/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":";;;AAAA,6EAAgE;AAChE,6CAAiC;AACjC,iCAAmC;AAGnC,MAAa,OAAO;IAApB;QACY,gBAAW,GAAG,CAAC,CAAC,CAAA;QAChB,cAAS,GAAG,CAAC,CAAC,CAAA;QACd,iBAAY,GAAG,IAAI,8BAAK,CAAC,EAAC,UAAU,EAAE,CAAC,EAAC,CAAC,CAAA;QACzC,qBAAgB,GAAG,IAAI,8BAAK,CAAC,EAAC,UAAU,EAAE,CAAC,EAAC,CAAC,CAAA;QAC7C,kBAAa,GAAG,IAAI,iCAAQ,CAAC,EAAC,YAAY,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,EAAC,CAAC,CAAA;IA6F3E,CAAC;IA3FG,cAAc,CAAC,MAAc;QACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;IACvD,CAAC;IAED,qBAAqB,CAAC,MAAc;QAChC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAA;QACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;IACjE,CAAC;IAED,cAAc,CAAC,SAAiB,EAAE,IAAY,EAAE,IAAa;QACzD,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,SAAS,GAAG,IAAI,CAAC,CAAA;QACnD,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACvD,CAAC;IAED,aAAa,CACT,SAAiB,EACjB,aAAqB,EACrB,qBAA6B,EAC7B,mBAA2B;QAE3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,EAAE,mBAAmB,CAAC,CAAA;QAC7E,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,qBAAqB,EAAE,mBAAmB,CAAC,CAAA;IAC9F,CAAC;IAED,cAAc;QACV,OAAO,IAAI,CAAC,WAAW,CAAA;IAC3B,CAAC;IAED,qBAAqB;QACjB,OAAO,IAAI,CAAC,SAAS,CAAA;IACzB,CAAC;IAED,YAAY;QACR,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAA;IACrC,CAAC;IAED,iBAAiB;QACb,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAAA;IACnC,CAAC;IAED,YAAY;QACR,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAA;IACrC,CAAC;IAED,eAAe;QACX,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAA;IACpC,CAAC;IAED,mBAAmB;QACf,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAA;IACxC,CAAC;IAED,aAAa;QACT,OAAO,GAAG,IAAI,CAAC,SAAS,MAAM,IAAI,CAAC,WAAW,IAAI;YAClD,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,eAAe;YACvD,YAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe;YAC7D,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC,cAAc;YACvD,QAAQ,IAAA,mBAAY,EAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,EAAE,CAAA;IACpD,CAAC;IAED,OAAO;QACH,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,4BAA4B;YAClC,IAAI,EAAE,iCAAiC;YACvC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;SAChD,CAAC,CAAA;QAEF,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE,sBAAsB;YAC5B,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;SACvD,CAAC,CAAA;QAEF,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,yCAAyC;YAC/C,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;SACjD,CAAC,CAAA;QAEF,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,gCAAgC;YACtC,IAAI,EAAE,sGAAsG;YAC5G,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;SACnD,CAAC,CAAA;QAEF,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE,gCAAgC;YACtC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;SAC9C,CAAC,CAAA;IACN,CAAC;CACJ;AAlGD,0BAkGC;AAGD,SAAS,OAAO,CAAC,EAAgB;IAC7B,OAAO;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAA;IAClB,CAAC,CAAA;AACL,CAAC"}

package/lib/run.d.ts

@@ -0,0 +1,36 @@
+import { Database, HashAndHeight } from './database';
+import { DataSource } from './datasource';
+export interface DataHandlerContext<Block, Store> {
+    /**
+     * Storage interface provided by the database
+     */
+    store: Store;
+    /**
+     * List of blocks to map and process
+     */
+    blocks: Block[];
+    /**
+     * Signals, that the processor is near the head of the chain.
+     */
+    isHead: boolean;
+}
+interface BlockBase {
+    header: HashAndHeight;
+}
+/**
+ * Run data processing.
+ *
+ * This method assumes full control over the current OS process as
+ * it terminates the entire program in case of error or
+ * at the end of data processing.
+ *
+ * @param src - data source to ingest data from
+ *
+ * @param db - database is responsible for providing storage API to data handler
+ * and persisting mapping progress and status.
+ *
+ * @param dataHandler - The data handler, see {@link DataHandlerContext} for an API available to the handler.
+ */
+export declare function run<Block extends BlockBase, Store>(src: DataSource<Block>, db: Database<Store>, dataHandler: (ctx: DataHandlerContext<Block, Store>) => Promise<void>): void;
+export {};
+//# sourceMappingURL=run.d.ts.map

package/lib/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,QAAQ,EAAE,aAAa,EAAC,MAAM,YAAY,CAAA;AAClD,OAAO,EAAC,UAAU,EAAC,MAAM,cAAc,CAAA;AAQvC,MAAM,WAAW,kBAAkB,CAAC,KAAK,EAAE,KAAK;IAC5C;;OAEG;IACH,KAAK,EAAE,KAAK,CAAA;IACZ;;OAEG;IACH,MAAM,EAAE,KAAK,EAAE,CAAA;IACf;;OAEG;IACH,MAAM,EAAE,OAAO,CAAA;CAClB;AAGD,UAAU,SAAS;IACf,MAAM,EAAE,aAAa,CAAA;CACxB;AAGD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,GAAG,CAAC,KAAK,SAAS,SAAS,EAAE,KAAK,EAC9C,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,EACtB,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,EACnB,WAAW,EAAE,CAAC,GAAG,EAAE,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GACtE,IAAI,CAMN"}

package/lib/run.js

@@ -0,0 +1,168 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.run = void 0;
+const logger_1 = require("@subsquid/logger");
+const util_internal_1 = require("@subsquid/util-internal");
+const util_internal_prometheus_server_1 = require("@subsquid/util-internal-prometheus-server");
+const prom = __importStar(require("prom-client"));
+const metrics_1 = require("./metrics");
+const util_1 = require("./util");
+const log = (0, logger_1.createLogger)('sqd:batch-processor');
+/**
+ * Run data processing.
+ *
+ * This method assumes full control over the current OS process as
+ * it terminates the entire program in case of error or
+ * at the end of data processing.
+ *
+ * @param src - data source to ingest data from
+ *
+ * @param db - database is responsible for providing storage API to data handler
+ * and persisting mapping progress and status.
+ *
+ * @param dataHandler - The data handler, see {@link DataHandlerContext} for an API available to the handler.
+ */
+function run(src, db, dataHandler) {
+    (0, util_internal_1.runProgram)(() => {
+        return new Processor(src, db, dataHandler).run();
+    }, err => {
+        log.fatal(err);
+    });
+}
+exports.run = run;
+class Processor {
+    constructor(src, db, handler) {
+        this.src = src;
+        this.db = db;
+        this.handler = handler;
+        this.metrics = new metrics_1.Metrics();
+        this.hasStatusNews = false;
+        this.chainHeight = new util_internal_1.Throttler(() => this.src.getFinalizedHeight(), 30000);
+    }
+    async run() {
+        let state = await this.db.connect();
+        if (state.height >= 0) {
+            log.info(`last processed final block was ${state.height}`);
+        }
+        await this.assertWeAreOnTheSameChain(state);
+        await this.initMetrics(state);
+        for await (let blocks of this.src.getBlockStream(state.height + 1)) {
+            if (blocks.length > 0) {
+                state = await this.processBatch(state, blocks);
+            }
+        }
+        this.reportFinalStatus();
+    }
+    async assertWeAreOnTheSameChain(state) {
+        if (state.height < 0)
+            return;
+        let hash = await this.src.getBlockHash(state.height);
+        if (state.hash === hash)
+            return;
+        throw new Error(`already indexed block ${(0, util_1.formatHead)(state)} was not found on chain`);
+    }
+    async initMetrics(state) {
+        await this.updateProgressMetrics(await this.chainHeight.get(), state);
+        let port = process.env.PROCESSOR_PROMETHEUS_PORT || process.env.PROMETHEUS_PORT;
+        if (port == null)
+            return;
+        prom.collectDefaultMetrics();
+        this.metrics.install();
+        let server = await (0, util_internal_prometheus_server_1.createPrometheusServer)(prom.register, port);
+        log.info(`prometheus metrics are served on port ${server.port}`);
+    }
+    updateProgressMetrics(chainHeight, state, time) {
+        this.metrics.setChainHeight(chainHeight);
+        this.metrics.setLastProcessedBlock(state.height);
+        let left;
+        let processed;
+        if (this.src.getBlocksCountInRange) {
+            left = this.src.getBlocksCountInRange({
+                from: this.metrics.getLastProcessedBlock() + 1,
+                to: this.metrics.getChainHeight()
+            });
+            processed = this.src.getBlocksCountInRange({
+                from: 0,
+                to: this.metrics.getChainHeight()
+            }) - left;
+        }
+        else {
+            left = this.metrics.getChainHeight() - this.metrics.getLastProcessedBlock();
+            processed = this.metrics.getLastProcessedBlock();
+        }
+        this.metrics.updateProgress(processed, left, time);
+    }
+    async processBatch(prevHead, blocks) {
+        let chainHeight = await this.chainHeight.get();
+        let nextHead = {
+            hash: (0, util_internal_1.last)(blocks).header.hash,
+            height: (0, util_internal_1.last)(blocks).header.height
+        };
+        let isOnTop = nextHead.height >= chainHeight;
+        let mappingStartTime = process.hrtime.bigint();
+        await this.db.transact({
+            prevHead,
+            nextHead,
+            isOnTop
+        }, store => {
+            return this.handler({
+                store,
+                blocks,
+                isHead: isOnTop
+            });
+        });
+        let mappingEndTime = process.hrtime.bigint();
+        this.updateProgressMetrics(chainHeight, nextHead, mappingEndTime);
+        this.metrics.registerBatch(blocks.length, (0, util_1.getItemsCount)(blocks), mappingStartTime, mappingEndTime);
+        this.reportStatus();
+        return nextHead;
+    }
+    reportStatus() {
+        if (this.statusReportTimer == null) {
+            log.info(this.metrics.getStatusLine());
+            this.statusReportTimer = setTimeout(() => {
+                this.statusReportTimer = undefined;
+                if (this.hasStatusNews) {
+                    this.hasStatusNews = false;
+                    this.reportStatus();
+                }
+            }, 5000);
+        }
+        else {
+            this.hasStatusNews = true;
+        }
+    }
+    reportFinalStatus() {
+        if (this.statusReportTimer != null) {
+            clearTimeout(this.statusReportTimer);
+        }
+        if (this.hasStatusNews) {
+            this.hasStatusNews = false;
+            log.info(this.metrics.getStatusLine());
+        }
+    }
+}
+//# sourceMappingURL=run.js.map

package/lib/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA6C;AAC7C,2DAAmE;AACnE,+FAAgF;AAChF,kDAAmC;AAGnC,uCAAiC;AACjC,iCAAgD;AAGhD,MAAM,GAAG,GAAG,IAAA,qBAAY,EAAC,qBAAqB,CAAC,CAAA;AAwB/C;;;;;;;;;;;;;GAaG;AACH,SAAgB,GAAG,CACf,GAAsB,EACtB,EAAmB,EACnB,WAAqE;IAErE,IAAA,0BAAU,EAAC,GAAG,EAAE;QACZ,OAAO,IAAI,SAAS,CAAC,GAAG,EAAE,EAAE,EAAE,WAAW,CAAC,CAAC,GAAG,EAAE,CAAA;IACpD,CAAC,EAAE,GAAG,CAAC,EAAE;QACL,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAClB,CAAC,CAAC,CAAA;AACN,CAAC;AAVD,kBAUC;AAGD,MAAM,SAAS;IAMX,YACY,GAAkB,EAClB,EAAe,EACf,OAAyD;QAFzD,QAAG,GAAH,GAAG,CAAe;QAClB,OAAE,GAAF,EAAE,CAAa;QACf,YAAO,GAAP,OAAO,CAAkD;QAR7D,YAAO,GAAG,IAAI,iBAAO,EAAE,CAAA;QAGvB,kBAAa,GAAG,KAAK,CAAA;QAOzB,IAAI,CAAC,WAAW,GAAG,IAAI,yBAAS,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,EAAE,KAAM,CAAC,CAAA;IACjF,CAAC;IAED,KAAK,CAAC,GAAG;QACL,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAA;QACnC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,kCAAkC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,MAAM,IAAI,CAAC,yBAAyB,CAAC,KAAK,CAAC,CAAA;QAC3C,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;QAE7B,IAAI,KAAK,EAAE,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YACjE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;YAClD,CAAC;QACL,CAAC;QAED,IAAI,CAAC,iBAAiB,EAAE,CAAA;IAC5B,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,KAAoB;QACxD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,OAAM;QAC5B,IAAI,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QACpD,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI;YAAE,OAAM;QAC/B,MAAM,IAAI,KAAK,CACX,yBAAyB,IAAA,iBAAU,EAAC,KAAK,CAAC,yBAAyB,CACtE,CAAA;IACL,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAAoB;QAC1C,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;QACrE,IAAI,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAA;QAC/E,IAAI,IAAI,IAAI,IAAI;YAAE,OAAM;QACxB,IAAI,CAAC,qBAAqB,EAAE,CAAA;QAC5B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACtB,IAAI,MAAM,GAAG,MAAM,IAAA,wDAAsB,EAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;QAC9D,GAAG,CAAC,IAAI,CAAC,yCAAyC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IACpE,CAAC;IAEO,qBAAqB,CAAC,WAAmB,EAAE,KAAoB,EAAE,IAAa;QAClF,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAChD,IAAI,IAAY,CAAA;QAChB,IAAI,SAAiB,CAAA;QACrB,IAAI,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC;YACjC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,CAAC;gBAClC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,GAAG,CAAC;gBAC9C,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE;aACpC,CAAC,CAAA;YACF,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,CAAC;gBACvC,IAAI,EAAE,CAAC;gBACP,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE;aACpC,CAAC,GAAG,IAAI,CAAA;QACb,CAAC;aAAM,CAAC;YACJ,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAA;YAC3E,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAA;QACpD,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;IACtD,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,QAAuB,EAAE,MAAW;QAC3D,IAAI,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAA;QAE9C,IAAI,QAAQ,GAAG;YACX,IAAI,EAAE,IAAA,oBAAI,EAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI;YAC9B,MAAM,EAAE,IAAA,oBAAI,EAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM;SACrC,CAAA;QAED,IAAI,OAAO,GAAG,QAAQ,CAAC,MAAM,IAAI,WAAW,CAAA;QAE5C,IAAI,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QAE9C,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC;YACnB,QAAQ;YACR,QAAQ;YACR,OAAO;SACV,EAAE,KAAK,CAAC,EAAE;YACP,OAAO,IAAI,CAAC,OAAO,CAAC;gBAChB,KAAK;gBACL,MAAM;gBACN,MAAM,EAAE,OAAO;aAClB,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;QAEF,IAAI,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QAE5C,IAAI,CAAC,qBAAqB,CAAC,WAAW,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAA;QACjE,IAAI,CAAC,OAAO,CAAC,aAAa,CACtB,MAAM,CAAC,MAAM,EACb,IAAA,oBAAa,EAAC,MAAM,CAAC,EACrB,gBAAgB,EAChB,cAAc,CACjB,CAAA;QAED,IAAI,CAAC,YAAY,EAAE,CAAA;QAEnB,OAAO,QAAQ,CAAA;IACnB,CAAC;IAEO,YAAY;QAChB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;YACtC,IAAI,CAAC,iBAAiB,GAAG,UAAU,CAAC,GAAG,EAAE;gBACrC,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAA;gBAClC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;oBACrB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;oBAC1B,IAAI,CAAC,YAAY,EAAE,CAAA;gBACvB,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,CAAA;QACZ,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;QAC7B,CAAC;IACL,CAAC;IAEO,iBAAiB;QACrB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACjC,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACxC,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;YAC1B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;QAC1C,CAAC;IACL,CAAC;CACJ"}

package/lib/util.d.ts

@@ -0,0 +1,6 @@
+import { HashAndHeight } from './database';
+export declare function timeInterval(seconds: number): string;
+export declare function getItemsCount(blocks: any[]): number;
+export declare function formatHead(head: HashAndHeight): string;
+export declare function shortHash(hash: string): string;
+//# sourceMappingURL=util.d.ts.map

package/lib/util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,YAAY,CAAA;AAGxC,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAWpD;AAGD,wBAAgB,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,CAWnD;AAGD,wBAAgB,UAAU,CAAC,IAAI,EAAE,aAAa,GAAG,MAAM,CAEtD;AAGD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAM9C"}

package/lib/util.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.shortHash = exports.formatHead = exports.getItemsCount = exports.timeInterval = void 0;
+function timeInterval(seconds) {
+    if (seconds < 60) {
+        return Math.round(seconds) + 's';
+    }
+    let minutes = Math.ceil(seconds / 60);
+    if (minutes < 60) {
+        return minutes + 'm';
+    }
+    let hours = Math.floor(minutes / 60);
+    minutes = minutes - hours * 60;
+    return hours + 'h ' + minutes + 'm';
+}
+exports.timeInterval = timeInterval;
+function getItemsCount(blocks) {
+    let count = 0;
+    for (let block of blocks) {
+        for (let key in block) {
+            let val = block[key];
+            if (Array.isArray(val)) {
+                count += val.length;
+            }
+        }
+    }
+    return count;
+}
+exports.getItemsCount = getItemsCount;
+function formatHead(head) {
+    return `${head.height}#${shortHash(head.hash)}`;
+}
+exports.formatHead = formatHead;
+function shortHash(hash) {
+    if (hash.startsWith('0x')) {
+        return hash.slice(2, 7);
+    }
+    else {
+        return hash.slice(0, 5);
+    }
+}
+exports.shortHash = shortHash;
+//# sourceMappingURL=util.js.map

package/lib/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;AAGA,SAAgB,YAAY,CAAC,OAAe;IACxC,IAAI,OAAO,GAAG,EAAE,EAAE,CAAC;QACf,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,GAAG,CAAA;IACpC,CAAC;IACD,IAAI,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,GAAC,EAAE,CAAC,CAAA;IACnC,IAAI,OAAO,GAAG,EAAE,EAAE,CAAC;QACf,OAAQ,OAAO,GAAC,GAAG,CAAA;IACvB,CAAC;IACD,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAA;IACpC,OAAO,GAAG,OAAO,GAAG,KAAK,GAAG,EAAE,CAAA;IAC9B,OAAO,KAAK,GAAG,IAAI,GAAG,OAAO,GAAG,GAAG,CAAA;AACvC,CAAC;AAXD,oCAWC;AAGD,SAAgB,aAAa,CAAC,MAAa;IACvC,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACvB,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;YACpB,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAA;YACpB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,KAAK,IAAI,GAAG,CAAC,MAAM,CAAA;YACvB,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAA;AAChB,CAAC;AAXD,sCAWC;AAGD,SAAgB,UAAU,CAAC,IAAmB;IAC1C,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAA;AACnD,CAAC;AAFD,gCAEC;AAGD,SAAgB,SAAS,CAAC,IAAY;IAClC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC3B,CAAC;SAAM,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC3B,CAAC;AACL,CAAC;AAND,8BAMC"}

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@subsquid/batch-processor",
+  "version": "0.0.0",
+  "description": "ETL processor",
+  "license": "GPL-3.0-or-later",
+  "repository": "git@github.com:subsquid/squid.git",
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "files": [
+    "lib",
+    "src"
+  ],
+  "main": "lib/index.js",
+  "dependencies": {
+    "@subsquid/logger": "^1.3.3",
+    "@subsquid/util-internal": "^3.2.0",
+    "@subsquid/util-internal-counters": "^1.3.2",
+    "@subsquid/util-internal-prometheus-server": "^1.3.0",
+    "@subsquid/util-internal-range": "^0.3.0",
+    "prom-client": "^14.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "^18.18.14",
+    "typescript": "~5.3.2"
+  },
+  "scripts": {
+    "build": "rm -rf lib && tsc"
+  }
+}

package/src/database.ts

@@ -0,0 +1,53 @@
+/**
+ * Database is responsible for providing a persistent storage for data handlers
+ * and keeping the processor progress and status.
+ */
+export type Database<S> = FinalDatabase<S> | HotDatabase<S>
+
+
+export interface FinalTxInfo {
+    prevHead: HashAndHeight
+    nextHead: HashAndHeight
+    isOnTop: boolean
+}
+
+
+export interface FinalDatabase<S> {
+    supportsHotBlocks?: false
+    connect(): Promise<HashAndHeight>
+    transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>
+}
+
+
+export interface HotTxInfo {
+    finalizedHead: HashAndHeight
+    baseHead: HashAndHeight
+    newBlocks: HashAndHeight[]
+}
+
+
+export interface HotDatabase<S> {
+    supportsHotBlocks: true
+    connect(): Promise<HotDatabaseState>
+    transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>
+    /**
+     * @deprecated
+     */
+    transactHot(info: HotTxInfo, cb: (store: S, block: HashAndHeight) => Promise<void>): Promise<void>
+
+    transactHot2?(
+        info: HotTxInfo,
+        cb: (store: S, blockSliceStart: number, blockSliceEnd: number) => Promise<void>
+    ): Promise<void>
+}
+
+
+export interface HotDatabaseState extends HashAndHeight {
+    top: HashAndHeight[]
+}
+
+
+export interface HashAndHeight {
+    height: number
+    hash: string
+}

package/src/datasource.ts

@@ -0,0 +1,9 @@
+import type {FiniteRange} from '@subsquid/util-internal-range'
+
+
+export interface DataSource<B> {
+    getFinalizedHeight(): Promise<number>
+    getBlockHash(height: number): Promise<string | undefined>
+    getBlockStream(fromBlock?: number): AsyncIterable<B[]>
+    getBlocksCountInRange?(range: FiniteRange): number
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from './database'
+export * from './datasource'
+export * from './run'

package/src/metrics.ts

@@ -0,0 +1,111 @@
+import {Progress, Speed} from '@subsquid/util-internal-counters'
+import {Gauge} from 'prom-client'
+import {timeInterval} from './util'
+
+
+export class Metrics {
+    private chainHeight = -1
+    private lastBlock = -1
+    private mappingSpeed = new Speed({windowSize: 5})
+    private mappingItemSpeed = new Speed({windowSize: 5})
+    private blockProgress = new Progress({initialValue: 0, windowSize: 20})
+
+    setChainHeight(height: number): void {
+        this.chainHeight = Math.max(height, this.lastBlock)
+    }
+
+    setLastProcessedBlock(height: number): void {
+        this.lastBlock = height
+        this.chainHeight = Math.max(this.chainHeight, this.lastBlock)
+    }
+
+    updateProgress(processed: number, left: number, time?: bigint): void {
+        this.blockProgress.setTargetValue(processed + left)
+        this.blockProgress.setCurrentValue(processed, time)
+    }
+
+    registerBatch(
+        batchSize: number,
+        batchItemSize: number,
+        batchMappingStartTime: bigint,
+        batchMappingEndTime: bigint,
+    ): void {
+        this.mappingSpeed.push(batchSize, batchMappingStartTime, batchMappingEndTime)
+        this.mappingItemSpeed.push(batchItemSize || 1, batchMappingStartTime, batchMappingEndTime)
+    }
+
+    getChainHeight(): number {
+        return this.chainHeight
+    }
+
+    getLastProcessedBlock(): number {
+        return this.lastBlock
+    }
+
+    getSyncSpeed(): number {
+        return this.blockProgress.speed()
+    }
+
+    getSyncEtaSeconds(): number {
+        return this.blockProgress.eta()
+    }
+
+    getSyncRatio(): number {
+        return this.blockProgress.ratio()
+    }
+
+    getMappingSpeed(): number {
+        return this.mappingSpeed.speed()
+    }
+
+    getMappingItemSpeed(): number {
+        return this.mappingItemSpeed.speed()
+    }
+
+    getStatusLine(): string {
+        return `${this.lastBlock} / ${this.chainHeight}, ` +
+        `rate: ${Math.round(this.getSyncSpeed())} blocks/sec, ` +
+        `mapping: ${Math.round(this.getMappingSpeed())} blocks/sec, ` +
+        `${Math.round(this.getMappingItemSpeed())} items/sec, ` +
+        `eta: ${timeInterval(this.getSyncEtaSeconds())}`
+    }
+
+    install(): void {
+        new Gauge({
+            name: 'sqd_processor_chain_height',
+            help: 'Chain height of the data source',
+            collect: collect(() => this.getChainHeight())
+        })
+
+        new Gauge({
+            name: 'sqd_processor_last_block',
+            help: 'Last processed block',
+            collect: collect(() => this.getLastProcessedBlock())
+        })
+
+        new Gauge({
+            name: 'sqd_processor_mapping_blocks_per_second',
+            help: 'Mapping performance',
+            collect: collect(() => this.getMappingSpeed())
+        })
+
+        new Gauge({
+            name: 'sqd_processor_sync_eta_seconds',
+            help: 'Estimated time until all required blocks will be processed or until the chain height will be reached',
+            collect: collect(() => this.getSyncEtaSeconds())
+        })
+
+        new Gauge({
+            name: 'sqd_processor_sync_ratio',
+            help: 'Percentage of processed blocks',
+            collect: collect(() => this.getSyncRatio())
+        })
+    }
+}
+
+
+function collect(fn: () => number) {
+    return function(this: Gauge<string>) {
+        this.set(fn())
+    }
+}

package/src/run.ts

@@ -0,0 +1,197 @@
+import {createLogger} from '@subsquid/logger'
+import {last, runProgram, Throttler} from '@subsquid/util-internal'
+import {createPrometheusServer} from '@subsquid/util-internal-prometheus-server'
+import * as prom from 'prom-client'
+import {Database, HashAndHeight} from './database'
+import {DataSource} from './datasource'
+import {Metrics} from './metrics'
+import {formatHead, getItemsCount} from './util'
+
+
+const log = createLogger('sqd:batch-processor')
+
+
+export interface DataHandlerContext<Block, Store> {
+    /**
+     * Storage interface provided by the database
+     */
+    store: Store
+    /**
+     * List of blocks to map and process
+     */
+    blocks: Block[]
+    /**
+     * Signals, that the processor is near the head of the chain.
+     */
+    isHead: boolean
+}
+
+
+interface BlockBase {
+    header: HashAndHeight
+}
+
+
+/**
+ * Run data processing.
+ *
+ * This method assumes full control over the current OS process as
+ * it terminates the entire program in case of error or
+ * at the end of data processing.
+ *
+ * @param src - data source to ingest data from
+ *
+ * @param db - database is responsible for providing storage API to data handler
+ * and persisting mapping progress and status.
+ *
+ * @param dataHandler - The data handler, see {@link DataHandlerContext} for an API available to the handler.
+ */
+export function run<Block extends BlockBase, Store>(
+    src: DataSource<Block>,
+    db: Database<Store>,
+    dataHandler: (ctx: DataHandlerContext<Block, Store>) => Promise<void>
+): void {
+    runProgram(() => {
+        return new Processor(src, db, dataHandler).run()
+    }, err => {
+        log.fatal(err)
+    })
+}
+
+
+class Processor<B extends BlockBase, S> {
+    private metrics = new Metrics()
+    private chainHeight: Throttler<number>
+    private statusReportTimer?: any
+    private hasStatusNews = false
+
+    constructor(
+        private src: DataSource<B>,
+        private db: Database<S>,
+        private handler: (ctx: DataHandlerContext<B, S>) => Promise<void>
+    ) {
+        this.chainHeight = new Throttler(() => this.src.getFinalizedHeight(), 30_000)
+    }
+
+    async run(): Promise<void> {
+        let state = await this.db.connect()
+        if (state.height >= 0) {
+            log.info(`last processed final block was ${state.height}`)
+        }
+
+        await this.assertWeAreOnTheSameChain(state)
+        await this.initMetrics(state)
+
+        for await (let blocks of this.src.getBlockStream(state.height + 1)) {
+            if (blocks.length > 0) {
+                state = await this.processBatch(state, blocks)
+            }
+        }
+
+        this.reportFinalStatus()
+    }
+
+    private async assertWeAreOnTheSameChain(state: HashAndHeight): Promise<void> {
+        if (state.height < 0) return
+        let hash = await this.src.getBlockHash(state.height)
+        if (state.hash === hash) return
+        throw new Error(
+            `already indexed block ${formatHead(state)} was not found on chain`
+        )
+    }
+
+    private async initMetrics(state: HashAndHeight): Promise<void> {
+        await this.updateProgressMetrics(await this.chainHeight.get(), state)
+        let port = process.env.PROCESSOR_PROMETHEUS_PORT || process.env.PROMETHEUS_PORT
+        if (port == null) return
+        prom.collectDefaultMetrics()
+        this.metrics.install()
+        let server = await createPrometheusServer(prom.register, port)
+        log.info(`prometheus metrics are served on port ${server.port}`)
+    }
+
+    private updateProgressMetrics(chainHeight: number, state: HashAndHeight, time?: bigint): void {
+        this.metrics.setChainHeight(chainHeight)
+        this.metrics.setLastProcessedBlock(state.height)
+        let left: number
+        let processed: number
+        if (this.src.getBlocksCountInRange) {
+            left = this.src.getBlocksCountInRange({
+                from: this.metrics.getLastProcessedBlock() + 1,
+                to: this.metrics.getChainHeight()
+            })
+            processed = this.src.getBlocksCountInRange({
+                from: 0,
+                to: this.metrics.getChainHeight()
+            }) - left
+        } else {
+            left = this.metrics.getChainHeight() - this.metrics.getLastProcessedBlock()
+            processed = this.metrics.getLastProcessedBlock()
+        }
+        this.metrics.updateProgress(processed, left, time)
+    }
+
+    private async processBatch(prevHead: HashAndHeight, blocks: B[]): Promise<HashAndHeight> {
+        let chainHeight = await this.chainHeight.get()
+
+        let nextHead = {
+            hash: last(blocks).header.hash,
+            height: last(blocks).header.height
+        }
+
+        let isOnTop = nextHead.height >= chainHeight
+
+        let mappingStartTime = process.hrtime.bigint()
+
+        await this.db.transact({
+            prevHead,
+            nextHead,
+            isOnTop
+        }, store => {
+            return this.handler({
+                store,
+                blocks,
+                isHead: isOnTop
+            })
+        })
+
+        let mappingEndTime = process.hrtime.bigint()
+
+        this.updateProgressMetrics(chainHeight, nextHead, mappingEndTime)
+        this.metrics.registerBatch(
+            blocks.length,
+            getItemsCount(blocks),
+            mappingStartTime,
+            mappingEndTime
+        )
+
+        this.reportStatus()
+
+        return nextHead
+    }
+
+    private reportStatus(): void {
+        if (this.statusReportTimer == null) {
+            log.info(this.metrics.getStatusLine())
+            this.statusReportTimer = setTimeout(() => {
+                this.statusReportTimer = undefined
+                if (this.hasStatusNews) {
+                    this.hasStatusNews = false
+                    this.reportStatus()
+                }
+            }, 5000)
+        } else {
+            this.hasStatusNews = true
+        }
+    }
+
+    private reportFinalStatus(): void {
+        if (this.statusReportTimer != null) {
+            clearTimeout(this.statusReportTimer)
+        }
+        if (this.hasStatusNews) {
+            this.hasStatusNews = false
+            log.info(this.metrics.getStatusLine())
+        }
+    }
+}

package/src/util.ts

@@ -0,0 +1,43 @@
+import {HashAndHeight} from './database'
+
+
+export function timeInterval(seconds: number): string {
+    if (seconds < 60) {
+        return Math.round(seconds) + 's'
+    }
+    let minutes = Math.ceil(seconds/60)
+    if (minutes < 60) {
+        return  minutes+'m'
+    }
+    let hours = Math.floor(minutes / 60)
+    minutes = minutes - hours * 60
+    return hours + 'h ' + minutes + 'm'
+}
+
+
+export function getItemsCount(blocks: any[]): number {
+    let count = 0
+    for (let block of blocks) {
+        for (let key in block) {
+            let val = block[key]
+            if (Array.isArray(val)) {
+                count += val.length
+            }
+        }
+    }
+    return count
+}
+
+
+export function formatHead(head: HashAndHeight): string {
+    return `${head.height}#${shortHash(head.hash)}`
+}
+
+
+export function shortHash(hash: string): string {
+    if (hash.startsWith('0x')) {
+        return hash.slice(2, 7)
+    } else {
+        return hash.slice(0, 5)
+    }
+}