@subql/node-ethereum 2.0.1 → 2.0.2-1

package/dist/indexer/indexer.manager.js

@@ -26,90 +26,30 @@ const project_service_1 = require("./project.service");
 const sandbox_service_1 = require("./sandbox.service");
 const unfinalizedBlocks_service_1 = require("./unfinalizedBlocks.service");
 const logger = (0, node_core_1.getLogger)('indexer');
-let IndexerManager = class IndexerManager {
+let IndexerManager = class IndexerManager extends node_core_1.BaseIndexerManager {
     constructor(apiService, nodeConfig, sandboxService, dsProcessorService, dynamicDsService, unfinalizedBlocksService, projectService) {
-        this.apiService = apiService;
-        this.nodeConfig = nodeConfig;
-        this.sandboxService = sandboxService;
-        this.dsProcessorService = dsProcessorService;
-        this.dynamicDsService = dynamicDsService;
-        this.unfinalizedBlocksService = unfinalizedBlocksService;
+        super(apiService, nodeConfig, sandboxService, dsProcessorService, dynamicDsService, unfinalizedBlocksService, FilterTypeMap, ProcessorTypeMap);
         this.projectService = projectService;
-        logger.info('indexer manager start');
-    }
-    async indexBlock(blockContent, dataSources) {
-        const { block, blockHeight } = blockContent;
-        let dynamicDsCreated = false;
-        let reindexBlockHeight = null;
-        const filteredDataSources = this.filterDataSources(blockHeight, dataSources);
-        this.assertDataSources(filteredDataSources, blockHeight);
-        reindexBlockHeight = await this.processUnfinalizedBlocks(block);
-        // Only index block if we're not going to reindex
-        if (!reindexBlockHeight) {
-            await this.indexBlockData(blockContent, filteredDataSources, 
-            // eslint-disable-next-line @typescript-eslint/require-await
-            async (ds) => {
-                const vm = this.sandboxService.getDsProcessorWrapper(ds, this.apiService.api, blockContent);
-                // Inject function to create ds into vm
-                vm.freeze(async (templateName, args) => {
-                    const newDs = await this.dynamicDsService.createDynamicDatasource({
-                        templateName,
-                        args,
-                        startBlock: blockHeight,
-                    });
-                    // Push the newly created dynamic ds to be processed this block on any future extrinsics/events
-                    filteredDataSources.push(newDs);
-                    dynamicDsCreated = true;
-                }, 'createDynamicDatasource');
-                return vm;
-            });
-        }
-        return {
-            dynamicDsCreated,
-            blockHash: block.hash,
-            reindexBlockHeight,
-        };
+        this.isRuntimeDs = common_ethereum_1.isRuntimeDs;
+        this.isCustomDs = common_ethereum_1.isCustomDs;
+        this.updateCustomProcessor = ds_processor_service_1.asSecondLayerHandlerProcessor_1_0_0;
     }
     async start() {
         await this.projectService.init();
         logger.info('indexer manager started');
     }
-    async processUnfinalizedBlocks(block) {
-        if (this.nodeConfig.unfinalizedBlocks) {
-            return this.unfinalizedBlocksService.processUnfinalizedBlocks(block);
-        }
-        return null;
+    async indexBlock(block, dataSources) {
+        return super.internalIndexBlock(block, dataSources, () => this.getApi(block));
     }
-    filterDataSources(nextProcessingHeight, dataSources) {
-        let filteredDs;
-        filteredDs = dataSources.filter((ds) => ds.startBlock <= nextProcessingHeight);
-        if (filteredDs.length === 0) {
-            logger.error(`Did not find any matching datasouces`);
-            process.exit(1);
-        }
-        // perform filter for custom ds
-        filteredDs = filteredDs.filter((ds) => {
-            if ((0, common_ethereum_1.isCustomDs)(ds)) {
-                return this.dsProcessorService
-                    .getDsProcessor(ds)
-                    .dsFilterProcessor(ds, this.apiService.api);
-            }
-            else {
-                return true;
-            }
-        });
-        if (!filteredDs.length) {
-            logger.error(`Did not find any datasources with associated processor`);
-            process.exit(1);
-        }
-        return filteredDs;
+    getBlockHeight(block) {
+        return block.blockHeight;
     }
-    assertDataSources(ds, blockHeight) {
-        if (!ds.length) {
-            logger.error(`Your start block is greater than the current indexed block height in your database. Either change your startBlock (project.yaml) to <= ${blockHeight}
-         or delete your database and start again from the currently specified startBlock`);
-            process.exit(1);
-        }
+    getBlockHash(block) {
+        return block.block.hash;
+    }
+    // eslint-disable-next-line @typescript-eslint/require-await
+    async getApi(block) {
+        return this.apiService.api.getSafeApi(this.getBlockHeight(block));
     }
     async indexBlockData({ block, transactions }, dataSources, getVM) {
         var _a;
@@ -136,94 +76,19 @@ let IndexerManager = class IndexerManager {
             await this.indexData(common_ethereum_1.EthereumHandlerKind.Event, log, ds, getVM);
         }
     }
-    async indexData(kind, data, ds, getVM) {
-        let vm;
-        if ((0, common_ethereum_1.isRuntimeDs)(ds)) {
-            const handlers = ds.mapping.handlers.filter((h) => {
-                var _a;
-                return h.kind === kind &&
-                    FilterTypeMap[kind](data, h.filter, (_a = ds.options) === null || _a === void 0 ? void 0 : _a.address);
-            });
-            if (!handlers.length) {
-                return;
-            }
-            const parsedData = await DataAbiParser[kind](this.apiService.api)(data, ds);
-            for (const handler of handlers) {
-                vm = vm !== null && vm !== void 0 ? vm : (await getVM(ds));
-                this.nodeConfig.profiler
-                    ? await (0, node_core_1.profilerWrap)(vm.securedExec.bind(vm), 'handlerPerformance', handler.handler)(handler.handler, [parsedData])
-                    : await vm.securedExec(handler.handler, [parsedData]);
-            }
-        }
-        else if ((0, common_ethereum_1.isCustomDs)(ds)) {
-            const handlers = this.filterCustomDsHandlers(ds, data, ProcessorTypeMap[kind], (data, baseFilter) => {
-                switch (kind) {
-                    case common_ethereum_1.EthereumHandlerKind.Block:
-                        return block_ethereum_1.EthereumBlockWrapped.filterBlocksProcessor(data, baseFilter);
-                    case common_ethereum_1.EthereumHandlerKind.Call:
-                        return block_ethereum_1.EthereumBlockWrapped.filterTransactionsProcessor(data, baseFilter);
-                    case common_ethereum_1.EthereumHandlerKind.Event:
-                        return block_ethereum_1.EthereumBlockWrapped.filterLogsProcessor(data, baseFilter);
-                    default:
-                        throw new Error('Unsupported handler kind');
-                }
-            });
-            if (!handlers.length) {
-                return;
-            }
-            const parsedData = await DataAbiParser[kind](this.apiService.api)(data, ds);
-            for (const handler of handlers) {
-                vm = vm !== null && vm !== void 0 ? vm : (await getVM(ds));
-                await this.transformAndExecuteCustomDs(ds, vm, handler, parsedData);
-            }
-        }
-    }
-    filterCustomDsHandlers(ds, data, baseHandlerCheck, baseFilter) {
-        const plugin = this.dsProcessorService.getDsProcessor(ds);
-        return ds.mapping.handlers
-            .filter((handler) => {
-            const processor = plugin.handlerProcessors[handler.kind];
-            if (baseHandlerCheck(processor)) {
-                processor.baseFilter;
-                return baseFilter(data, processor.baseFilter);
-            }
-            return false;
-        })
-            .filter((handler) => {
-            const processor = (0, ds_processor_service_1.asSecondLayerHandlerProcessor_1_0_0)(plugin.handlerProcessors[handler.kind]);
-            try {
-                return processor.filterProcessor({
-                    filter: handler.filter,
-                    input: data,
-                    ds,
-                });
-            }
-            catch (e) {
-                logger.error(e, 'Failed to run ds processer filter.');
-                throw e;
-            }
-        });
+    async prepareFilteredData(kind, data, ds) {
+        return DataAbiParser[kind](this.apiService.api)(data, ds);
     }
-    async transformAndExecuteCustomDs(ds, vm, handler, data) {
-        const plugin = this.dsProcessorService.getDsProcessor(ds);
-        const assets = await this.dsProcessorService.getAssets(ds);
-        const processor = (0, ds_processor_service_1.asSecondLayerHandlerProcessor_1_0_0)(plugin.handlerProcessors[handler.kind]);
-        const transformedData = await processor
-            .transformer({
-            input: data,
-            ds,
-            api: this.apiService.api,
-            filter: handler.filter,
-            assets,
-        })
-            .catch((e) => {
-            logger.error(e, 'Failed to transform data with ds processor.');
-            throw e;
-        });
-        // We can not run this in parallel. the transformed data items may be dependent on one another.
-        // An example of this is with Acala EVM packing multiple EVM logs into a single Substrate event
-        for (const _data of transformedData) {
-            await vm.securedExec(handler.handler, [_data]);
+    baseCustomHandlerFilter(kind, data, baseFilter) {
+        switch (kind) {
+            case common_ethereum_1.EthereumHandlerKind.Block:
+                return block_ethereum_1.EthereumBlockWrapped.filterBlocksProcessor(data, baseFilter);
+            case common_ethereum_1.EthereumHandlerKind.Call:
+                return block_ethereum_1.EthereumBlockWrapped.filterTransactionsProcessor(data, baseFilter);
+            case common_ethereum_1.EthereumHandlerKind.Event:
+                return block_ethereum_1.EthereumBlockWrapped.filterLogsProcessor(data, baseFilter);
+            default:
+                throw new Error('Unsupported handler kind');
         }
     }
 };
@@ -257,7 +122,7 @@ const FilterTypeMap = {
 };
 const DataAbiParser = {
     [common_ethereum_1.EthereumHandlerKind.Block]: () => (data) => data,
-    [common_ethereum_1.EthereumHandlerKind.Event]: (api) => api.parseLog.bind(api),
-    [common_ethereum_1.EthereumHandlerKind.Call]: (api) => api.parseTransaction.bind(api),
+    [common_ethereum_1.EthereumHandlerKind.Event]: (api) => (data, ds) => api.parseLog(data, ds),
+    [common_ethereum_1.EthereumHandlerKind.Call]: (api) => (data, ds) => api.parseTransaction(data, ds),
 };
 //# sourceMappingURL=indexer.manager.js.map

package/dist/indexer/indexer.manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.manager.js","sourceRoot":"","sources":["../../src/indexer/indexer.manager.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,4DAUgC;AAChC,gDAS0B;AAU1B,+DAAkE;AAClE,oCAAwC;AACxC,iEAGgC;AAChC,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AAEvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAGzB,YACU,UAAsB,EACtB,UAAsB,EACtB,cAA8B,EAC9B,kBAAsC,EACtC,gBAAkC,EAClC,wBAAkD,EACvB,cAA8B;QANzD,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;QAC9B,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,6BAAwB,GAAxB,wBAAwB,CAA0B;QACvB,mBAAc,GAAd,cAAc,CAAgB;QAEjE,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACvC,CAAC;IAGK,AAAN,KAAK,CAAC,UAAU,CACd,YAAkC,EAClC,WAA6B;QAE7B,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,YAAY,CAAC;QAC5C,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,kBAAkB,GAAkB,IAAI,CAAC;QAE7C,MAAM,mBAAmB,GAAG,IAAI,CAAC,iBAAiB,CAChD,WAAW,EACX,WAAW,CACZ,CAAC;QAEF,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,EAAE,WAAW,CAAC,CAAC;QACzD,kBAAkB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAEhE,iDAAiD;QACjD,IAAI,CAAC,kBAAkB,EAAE;YACvB,MAAM,IAAI,CAAC,cAAc,CACvB,YAAY,EACZ,mBAAmB;YACnB,4DAA4D;YAC5D,KAAK,EAAE,EAAkB,EAAE,EAAE;gBAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAClD,EAAE,EACF,IAAI,CAAC,UAAU,CAAC,GAAG,EACnB,YAAY,CACb,CAAC;gBAEF,uCAAuC;gBACvC,EAAE,CAAC,MAAM,CACP,KAAK,EAAE,YAAoB,EAAE,IAA8B,EAAE,EAAE;oBAC7D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,uBAAuB,CAC/D;wBACE,YAAY;wBACZ,IAAI;wBACJ,UAAU,EAAE,WAAW;qBACxB,CACF,CAAC;oBACF,+FAA+F;oBAC/F,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAChC,gBAAgB,GAAG,IAAI,CAAC;gBAC1B,CAAC,EACD,yBAAyB,CAC1B,CAAC;gBAEF,OAAO,EAAE,CAAC;YACZ,CAAC,CACF,CAAC;SACH;QAED,OAAO;YACL,gBAAgB;YAChB,SAAS,EAAE,KAAK,CAAC,IAAI;YACrB,kBAAkB;SACnB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,KAAoB;QAEpB,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,OAAO,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;SACtE;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,iBAAiB,CACvB,oBAA4B,EAC5B,WAA6B;QAE7B,IAAI,UAA4B,CAAC;QAEjC,UAAU,GAAG,WAAW,CAAC,MAAM,CAC7B,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,oBAAoB,CAC9C,CAAC;QAEF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAC3B,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;YACrD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QACD,+BAA+B;QAC/B,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;YACpC,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;gBAClB,OAAO,IAAI,CAAC,kBAAkB;qBAC3B,cAAc,CAAC,EAAE,CAAC;qBAClB,iBAAiB,CAAC,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aAC/C;iBAAM;gBACL,OAAO,IAAI,CAAC;aACb;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;YACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,iBAAiB,CAAC,EAAoB,EAAE,WAAmB;QACjE,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;YACd,MAAM,CAAC,KAAK,CACV,0IAA0I,WAAW;yFACpE,CAClF,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,EAAE,KAAK,EAAE,YAAY,EAAwB,EAC7C,WAA6B,EAC7B,KAAqD;;QAErD,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAExD,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE;YAC7B,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YAEpD,KAAK,MAAM,GAAG,IAAI,MAAA,EAAE,CAAC,IAAI,mCAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;aAChD;SACF;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,KAAoB,EACpB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACnE;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,EAAuB,EACvB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SAC/D;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,GAAgB,EAChB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACjE;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,IAAO,EACP,IAAuC,EACvC,EAAkB,EAClB,KAAsD;QAEtD,IAAI,EAAkB,CAAC;QACvB,IAAI,IAAA,6BAAW,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,QAAQ,GAAI,EAAE,CAAC,OAAO,CAAC,QAAkC,CAAC,MAAM,CACpE,CAAC,CAAC,EAAE,EAAE;;gBACJ,OAAA,CAAC,CAAC,IAAI,KAAK,IAAI;oBACf,aAAa,CAAC,IAAI,CAAC,CACjB,IAAW,EACX,CAAC,CAAC,MAAa,EACf,MAAA,EAAE,CAAC,OAAO,0CAAE,OAAO,CACpB,CAAA;aAAA,CACJ,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;gBACpB,OAAO;aACR;YACD,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAC/D,IAAI,EACJ,EAAE,CACH,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,EAAE,GAAG,EAAE,aAAF,EAAE,cAAF,EAAE,GAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,IAAI,CAAC,UAAU,CAAC,QAAQ;oBACtB,CAAC,CAAC,MAAM,IAAA,wBAAY,EAChB,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,EACvB,oBAAoB,EACpB,OAAO,CAAC,OAAO,CAChB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;oBAClC,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;aACzD;SACF;aAAM,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAC1C,EAAE,EACF,IAAI,EACJ,gBAAgB,CAAC,IAAI,CAAC,EACtB,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE;gBACnB,QAAQ,IAAI,EAAE;oBACZ,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,qCAAoB,CAAC,qBAAqB,CAC/C,IAAqB,EACrB,UAAU,CACX,CAAC;oBACJ,KAAK,qCAAmB,CAAC,IAAI;wBAC3B,OAAO,qCAAoB,CAAC,2BAA2B,CACrD,IAA2B,EAC3B,UAAU,CACX,CAAC;oBACJ,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,qCAAoB,CAAC,mBAAmB,CAC7C,IAAmB,EACnB,UAAU,CACX,CAAC;oBACJ;wBACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;iBAC/C;YACH,CAAC,CACF,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;gBACpB,OAAO;aACR;YAED,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAC/D,IAAI,EACJ,EAAE,CACH,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,EAAE,GAAG,EAAE,aAAF,EAAE,cAAF,EAAE,GAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,MAAM,IAAI,CAAC,2BAA2B,CAAC,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;aACrE;SACF;IACH,CAAC;IAEO,sBAAsB,CAC5B,EAA8C,EAC9C,IAAuC,EACvC,gBAAqC,EACrC,UAGY;QAEZ,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAE1D,OAAO,EAAE,CAAC,OAAO,CAAC,QAAQ;aACvB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACzD,IAAI,gBAAgB,CAAC,SAAS,CAAC,EAAE;gBAC/B,SAAS,CAAC,UAAU,CAAC;gBAErB,OAAO,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;aAC/C;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,IAAA,0DAAmC,EACnD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CACvC,CAAC;YAEF,IAAI;gBACF,OAAO,SAAS,CAAC,eAAe,CAAC;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,KAAK,EAAE,IAAI;oBACX,EAAE;iBACH,CAAC,CAAC;aACJ;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oCAAoC,CAAC,CAAC;gBACtD,MAAM,CAAC,CAAC;aACT;QACH,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,EAA8C,EAC9C,EAAkB,EAClB,OAA2B,EAC3B,IAAuC;QAEvC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAE3D,MAAM,SAAS,GAAG,IAAA,0DAAmC,EACnD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CACvC,CAAC;QAEF,MAAM,eAAe,GAAG,MAAM,SAAS;aACpC,WAAW,CAAC;YACX,KAAK,EAAE,IAAI;YACX,EAAE;YACF,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG;YACxB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM;SACP,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,6CAA6C,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;QACV,CAAC,CAAC,CAAC;QAEL,+FAA+F;QAC/F,+FAA+F;QAC/F,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE;YACnC,MAAM,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;SAChD;IACH,CAAC;CACF,CAAA;AAvTO;IADL,IAAA,oBAAQ,EAAC,oBAAY,CAAC,IAAI,CAAC,QAAQ,CAAC;;;;gDAyDpC;AAxEU,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAWR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;qCANN,sBAAU;QACV,sBAAU;QACN,gCAAc;QACV,yCAAkB;QACpB,qCAAgB;QACR,oDAAwB;QACP,gCAAc;GAVxD,cAAc,CAuU1B;AAvUY,wCAAc;AA+U3B,MAAM,gBAAgB,GAAG;IACvB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,wCAAsB;CACnD,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,qBAAqB;IACvE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,mBAAmB;IACrE,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,qCAAoB,CAAC,2BAA2B;CAC7E,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,IAAmB,EAAE,EAAE,CAAC,IAAI;IAChE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,CAAC,GAAgB,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;IACzE,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,GAAgB,EAAE,EAAE,CAC/C,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC;CACjC,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  isBlockHandlerProcessor,\n  isCallHandlerProcessor,\n  isEventHandlerProcessor,\n  isCustomDs,\n  isRuntimeDs,\n  SubqlEthereumCustomDataSource,\n  SubqlCustomHandler,\n  EthereumHandlerKind,\n  EthereumRuntimeHandlerInputMap,\n} from '@subql/common-ethereum';\nimport {\n  NodeConfig,\n  getLogger,\n  profiler,\n  profilerWrap,\n  IndexerSandbox,\n  ProcessBlockResponse,\n  ApiService,\n  IIndexerManager,\n} from '@subql/node-core';\nimport {\n  EthereumTransaction,\n  EthereumLog,\n  SubqlRuntimeHandler,\n  EthereumBlockWrapper,\n  EthereumBlock,\n} from '@subql/types-ethereum';\nimport { SubqlProjectDs } from '../configure/SubqueryProject';\nimport { EthereumApi } from '../ethereum';\nimport { EthereumBlockWrapped } from '../ethereum/block.ethereum';\nimport { yargsOptions } from '../yargs';\nimport {\n  asSecondLayerHandlerProcessor_1_0_0,\n  DsProcessorService,\n} from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\nconst logger = getLogger('indexer');\n\n@Injectable()\nexport class IndexerManager\n  implements IIndexerManager<EthereumBlockWrapper, SubqlProjectDs>\n{\n  constructor(\n    private apiService: ApiService,\n    private nodeConfig: NodeConfig,\n    private sandboxService: SandboxService,\n    private dsProcessorService: DsProcessorService,\n    private dynamicDsService: DynamicDsService,\n    private unfinalizedBlocksService: UnfinalizedBlocksService,\n    @Inject('IProjectService') private projectService: ProjectService,\n  ) {\n    logger.info('indexer manager start');\n  }\n\n  @profiler(yargsOptions.argv.profiler)\n  async indexBlock(\n    blockContent: EthereumBlockWrapper,\n    dataSources: SubqlProjectDs[],\n  ): Promise<ProcessBlockResponse> {\n    const { block, blockHeight } = blockContent;\n    let dynamicDsCreated = false;\n    let reindexBlockHeight: number | null = null;\n\n    const filteredDataSources = this.filterDataSources(\n      blockHeight,\n      dataSources,\n    );\n\n    this.assertDataSources(filteredDataSources, blockHeight);\n    reindexBlockHeight = await this.processUnfinalizedBlocks(block);\n\n    // Only index block if we're not going to reindex\n    if (!reindexBlockHeight) {\n      await this.indexBlockData(\n        blockContent,\n        filteredDataSources,\n        // eslint-disable-next-line @typescript-eslint/require-await\n        async (ds: SubqlProjectDs) => {\n          const vm = this.sandboxService.getDsProcessorWrapper(\n            ds,\n            this.apiService.api,\n            blockContent,\n          );\n\n          // Inject function to create ds into vm\n          vm.freeze(\n            async (templateName: string, args?: Record<string, unknown>) => {\n              const newDs = await this.dynamicDsService.createDynamicDatasource(\n                {\n                  templateName,\n                  args,\n                  startBlock: blockHeight,\n                },\n              );\n              // Push the newly created dynamic ds to be processed this block on any future extrinsics/events\n              filteredDataSources.push(newDs);\n              dynamicDsCreated = true;\n            },\n            'createDynamicDatasource',\n          );\n\n          return vm;\n        },\n      );\n    }\n\n    return {\n      dynamicDsCreated,\n      blockHash: block.hash,\n      reindexBlockHeight,\n    };\n  }\n\n  async start(): Promise<void> {\n    await this.projectService.init();\n    logger.info('indexer manager started');\n  }\n\n  private async processUnfinalizedBlocks(\n    block: EthereumBlock,\n  ): Promise<number | null> {\n    if (this.nodeConfig.unfinalizedBlocks) {\n      return this.unfinalizedBlocksService.processUnfinalizedBlocks(block);\n    }\n    return null;\n  }\n\n  private filterDataSources(\n    nextProcessingHeight: number,\n    dataSources: SubqlProjectDs[],\n  ): SubqlProjectDs[] {\n    let filteredDs: SubqlProjectDs[];\n\n    filteredDs = dataSources.filter(\n      (ds) => ds.startBlock <= nextProcessingHeight,\n    );\n\n    if (filteredDs.length === 0) {\n      logger.error(`Did not find any matching datasouces`);\n      process.exit(1);\n    }\n    // perform filter for custom ds\n    filteredDs = filteredDs.filter((ds) => {\n      if (isCustomDs(ds)) {\n        return this.dsProcessorService\n          .getDsProcessor(ds)\n          .dsFilterProcessor(ds, this.apiService.api);\n      } else {\n        return true;\n      }\n    });\n\n    if (!filteredDs.length) {\n      logger.error(`Did not find any datasources with associated processor`);\n      process.exit(1);\n    }\n    return filteredDs;\n  }\n\n  private assertDataSources(ds: SubqlProjectDs[], blockHeight: number) {\n    if (!ds.length) {\n      logger.error(\n        `Your start block is greater than the current indexed block height in your database. Either change your startBlock (project.yaml) to <= ${blockHeight}\n         or delete your database and start again from the currently specified startBlock`,\n      );\n      process.exit(1);\n    }\n  }\n\n  private async indexBlockData(\n    { block, transactions }: EthereumBlockWrapper,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    await this.indexBlockContent(block, dataSources, getVM);\n\n    for (const tx of transactions) {\n      await this.indexTransaction(tx, dataSources, getVM);\n\n      for (const log of tx.logs ?? []) {\n        await this.indexEvent(log, dataSources, getVM);\n      }\n    }\n  }\n\n  private async indexBlockContent(\n    block: EthereumBlock,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Block, block, ds, getVM);\n    }\n  }\n\n  private async indexTransaction(\n    tx: EthereumTransaction,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Call, tx, ds, getVM);\n    }\n  }\n\n  private async indexEvent(\n    log: EthereumLog,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Event, log, ds, getVM);\n    }\n  }\n\n  private async indexData<K extends EthereumHandlerKind>(\n    kind: K,\n    data: EthereumRuntimeHandlerInputMap[K],\n    ds: SubqlProjectDs,\n    getVM: (ds: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    let vm: IndexerSandbox;\n    if (isRuntimeDs(ds)) {\n      const handlers = (ds.mapping.handlers as SubqlRuntimeHandler[]).filter(\n        (h) =>\n          h.kind === kind &&\n          FilterTypeMap[kind](\n            data as any,\n            h.filter as any,\n            ds.options?.address,\n          ),\n      );\n\n      if (!handlers.length) {\n        return;\n      }\n      const parsedData = await DataAbiParser[kind](this.apiService.api)(\n        data,\n        ds,\n      );\n\n      for (const handler of handlers) {\n        vm = vm ?? (await getVM(ds));\n        this.nodeConfig.profiler\n          ? await profilerWrap(\n              vm.securedExec.bind(vm),\n              'handlerPerformance',\n              handler.handler,\n            )(handler.handler, [parsedData])\n          : await vm.securedExec(handler.handler, [parsedData]);\n      }\n    } else if (isCustomDs(ds)) {\n      const handlers = this.filterCustomDsHandlers<K>(\n        ds,\n        data,\n        ProcessorTypeMap[kind],\n        (data, baseFilter) => {\n          switch (kind) {\n            case EthereumHandlerKind.Block:\n              return EthereumBlockWrapped.filterBlocksProcessor(\n                data as EthereumBlock,\n                baseFilter,\n              );\n            case EthereumHandlerKind.Call:\n              return EthereumBlockWrapped.filterTransactionsProcessor(\n                data as EthereumTransaction,\n                baseFilter,\n              );\n            case EthereumHandlerKind.Event:\n              return EthereumBlockWrapped.filterLogsProcessor(\n                data as EthereumLog,\n                baseFilter,\n              );\n            default:\n              throw new Error('Unsupported handler kind');\n          }\n        },\n      );\n\n      if (!handlers.length) {\n        return;\n      }\n\n      const parsedData = await DataAbiParser[kind](this.apiService.api)(\n        data,\n        ds,\n      );\n\n      for (const handler of handlers) {\n        vm = vm ?? (await getVM(ds));\n        await this.transformAndExecuteCustomDs(ds, vm, handler, parsedData);\n      }\n    }\n  }\n\n  private filterCustomDsHandlers<K extends EthereumHandlerKind>(\n    ds: SubqlEthereumCustomDataSource<string, any>,\n    data: EthereumRuntimeHandlerInputMap[K],\n    baseHandlerCheck: ProcessorTypeMap[K],\n    baseFilter: (\n      data: EthereumRuntimeHandlerInputMap[K],\n      baseFilter: any,\n    ) => boolean,\n  ): SubqlCustomHandler[] {\n    const plugin = this.dsProcessorService.getDsProcessor(ds);\n\n    return ds.mapping.handlers\n      .filter((handler) => {\n        const processor = plugin.handlerProcessors[handler.kind];\n        if (baseHandlerCheck(processor)) {\n          processor.baseFilter;\n\n          return baseFilter(data, processor.baseFilter);\n        }\n        return false;\n      })\n      .filter((handler) => {\n        const processor = asSecondLayerHandlerProcessor_1_0_0(\n          plugin.handlerProcessors[handler.kind],\n        );\n\n        try {\n          return processor.filterProcessor({\n            filter: handler.filter,\n            input: data,\n            ds,\n          });\n        } catch (e) {\n          logger.error(e, 'Failed to run ds processer filter.');\n          throw e;\n        }\n      });\n  }\n\n  private async transformAndExecuteCustomDs<K extends EthereumHandlerKind>(\n    ds: SubqlEthereumCustomDataSource<string, any>,\n    vm: IndexerSandbox,\n    handler: SubqlCustomHandler,\n    data: EthereumRuntimeHandlerInputMap[K],\n  ): Promise<void> {\n    const plugin = this.dsProcessorService.getDsProcessor(ds);\n    const assets = await this.dsProcessorService.getAssets(ds);\n\n    const processor = asSecondLayerHandlerProcessor_1_0_0(\n      plugin.handlerProcessors[handler.kind],\n    );\n\n    const transformedData = await processor\n      .transformer({\n        input: data,\n        ds,\n        api: this.apiService.api,\n        filter: handler.filter,\n        assets,\n      })\n      .catch((e) => {\n        logger.error(e, 'Failed to transform data with ds processor.');\n        throw e;\n      });\n\n    // We can not run this in parallel. the transformed data items may be dependent on one another.\n    // An example of this is with Acala EVM packing multiple EVM logs into a single Substrate event\n    for (const _data of transformedData) {\n      await vm.securedExec(handler.handler, [_data]);\n    }\n  }\n}\n\ntype ProcessorTypeMap = {\n  [EthereumHandlerKind.Block]: typeof isBlockHandlerProcessor;\n  [EthereumHandlerKind.Event]: typeof isEventHandlerProcessor;\n  [EthereumHandlerKind.Call]: typeof isCallHandlerProcessor;\n};\n\nconst ProcessorTypeMap = {\n  [EthereumHandlerKind.Block]: isBlockHandlerProcessor,\n  [EthereumHandlerKind.Event]: isEventHandlerProcessor,\n  [EthereumHandlerKind.Call]: isCallHandlerProcessor,\n};\n\nconst FilterTypeMap = {\n  [EthereumHandlerKind.Block]: EthereumBlockWrapped.filterBlocksProcessor,\n  [EthereumHandlerKind.Event]: EthereumBlockWrapped.filterLogsProcessor,\n  [EthereumHandlerKind.Call]: EthereumBlockWrapped.filterTransactionsProcessor,\n};\n\nconst DataAbiParser = {\n  [EthereumHandlerKind.Block]: () => (data: EthereumBlock) => data,\n  [EthereumHandlerKind.Event]: (api: EthereumApi) => api.parseLog.bind(api),\n  [EthereumHandlerKind.Call]: (api: EthereumApi) =>\n    api.parseTransaction.bind(api),\n};\n"]}
+{"version":3,"file":"indexer.manager.js","sourceRoot":"","sources":["../../src/indexer/indexer.manager.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,4DAUgC;AAChC,gDAQ0B;AAU1B,+DAAkE;AAElE,oCAAwC;AACxC,iEAGgC;AAChC,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AAEvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,8BASnC;IAKC,YACE,UAAsB,EACtB,UAAsB,EACtB,cAA8B,EAC9B,kBAAsC,EACtC,gBAAkC,EAClC,wBAAkD,EACf,cAA8B;QAEjE,KAAK,CACH,UAAU,EACV,UAAU,EACV,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,EACxB,aAAa,EACb,gBAAgB,CACjB,CAAC;QAXiC,mBAAc,GAAd,cAAc,CAAgB;QAXzD,gBAAW,GAAG,6BAAW,CAAC;QAC1B,eAAU,GAAG,4BAAU,CAAC;QACxB,0BAAqB,GAAG,0DAAmC,CAAC;IAqBtE,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACzC,CAAC;IAGK,AAAN,KAAK,CAAC,UAAU,CACd,KAA2B,EAC3B,WAAsC;QAEtC,OAAO,KAAK,CAAC,kBAAkB,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CACnB,CAAC;IACJ,CAAC;IAED,cAAc,CAAC,KAA2B;QACxC,OAAO,KAAK,CAAC,WAAW,CAAC;IAC3B,CAAC;IAED,YAAY,CAAC,KAA2B;QACtC,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;IAC1B,CAAC;IAED,4DAA4D;IACpD,KAAK,CAAC,MAAM,CAAC,KAA2B;QAC9C,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;IACpE,CAAC;IAES,KAAK,CAAC,cAAc,CAC5B,EAAE,KAAK,EAAE,YAAY,EAAwB,EAC7C,WAA6B,EAC7B,KAAqD;;QAErD,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAExD,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE;YAC7B,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YAEpD,KAAK,MAAM,GAAG,IAAI,MAAA,EAAE,CAAC,IAAI,mCAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;aAChD;SACF;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,KAAoB,EACpB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACnE;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,EAAuB,EACvB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SAC/D;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,GAAgB,EAChB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACjE;IACH,CAAC;IAES,KAAK,CAAC,mBAAmB,CACjC,IAAyB,EACzB,IAAS,EACT,EAA0B;QAE1B,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC5D,CAAC;IAES,uBAAuB,CAC/B,IAAyB,EACzB,IAAS,EACT,UAAe;QAEf,QAAQ,IAAI,EAAE;YACZ,KAAK,qCAAmB,CAAC,KAAK;gBAC5B,OAAO,qCAAoB,CAAC,qBAAqB,CAC/C,IAAqB,EACrB,UAAU,CACX,CAAC;YACJ,KAAK,qCAAmB,CAAC,IAAI;gBAC3B,OAAO,qCAAoB,CAAC,2BAA2B,CACrD,IAA2B,EAC3B,UAAU,CACX,CAAC;YACJ,KAAK,qCAAmB,CAAC,KAAK;gBAC5B,OAAO,qCAAoB,CAAC,mBAAmB,CAC7C,IAAmB,EACnB,UAAU,CACX,CAAC;YACJ;gBACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;SAC/C;IACH,CAAC;CACF,CAAA;AArGO;IADL,IAAA,oBAAQ,EAAC,oBAAY,CAAC,IAAI,CAAC,QAAQ,CAAC;;;;gDAQpC;AAhDU,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAsBR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;qCANd,sBAAU;QACV,sBAAU;QACN,gCAAc;QACV,yCAAkB;QACpB,qCAAgB;QACR,oDAAwB;QACC,gCAAc;GArBxD,cAAc,CA8I1B;AA9IY,wCAAc;AAsJ3B,MAAM,gBAAgB,GAAG;IACvB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,wCAAsB;CACnD,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,qBAAqB;IACvE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,mBAAmB;IACrE,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,qCAAoB,CAAC,2BAA2B;CAC7E,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,IAAmB,EAAE,EAAE,CAAC,IAAI;IAChE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EACzB,CAAC,GAAgB,EAAE,EAAE,CAAC,CAAC,IAAiB,EAAE,EAA0B,EAAE,EAAE,CACtE,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;IAC1B,CAAC,qCAAmB,CAAC,IAAI,CAAC,EACxB,CAAC,GAAgB,EAAE,EAAE,CACrB,CAAC,IAAyB,EAAE,EAA0B,EAAE,EAAE,CACxD,GAAG,CAAC,gBAAgB,CAAC,IAAI,EAAE,EAAE,CAAC;CACnC,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  isBlockHandlerProcessor,\n  isCallHandlerProcessor,\n  isEventHandlerProcessor,\n  isCustomDs,\n  isRuntimeDs,\n  SubqlEthereumCustomDataSource,\n  EthereumHandlerKind,\n  EthereumRuntimeHandlerInputMap,\n  SubqlEthereumDataSource,\n} from '@subql/common-ethereum';\nimport {\n  NodeConfig,\n  getLogger,\n  profiler,\n  IndexerSandbox,\n  ProcessBlockResponse,\n  BaseIndexerManager,\n  ApiService,\n} from '@subql/node-core';\nimport {\n  EthereumTransaction,\n  EthereumLog,\n  EthereumBlockWrapper,\n  EthereumBlock,\n  SubqlRuntimeDatasource,\n} from '@subql/types-ethereum';\nimport { SubqlProjectDs } from '../configure/SubqueryProject';\nimport { EthereumApi } from '../ethereum';\nimport { EthereumBlockWrapped } from '../ethereum/block.ethereum';\nimport SafeEthProvider from '../ethereum/safe-api';\nimport { yargsOptions } from '../yargs';\nimport {\n  asSecondLayerHandlerProcessor_1_0_0,\n  DsProcessorService,\n} from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\nconst logger = getLogger('indexer');\n\n@Injectable()\nexport class IndexerManager extends BaseIndexerManager<\n  ApiService,\n  SafeEthProvider,\n  EthereumBlockWrapper,\n  SubqlEthereumDataSource,\n  SubqlEthereumCustomDataSource,\n  typeof FilterTypeMap,\n  typeof ProcessorTypeMap,\n  EthereumRuntimeHandlerInputMap\n> {\n  protected isRuntimeDs = isRuntimeDs;\n  protected isCustomDs = isCustomDs;\n  protected updateCustomProcessor = asSecondLayerHandlerProcessor_1_0_0;\n\n  constructor(\n    apiService: ApiService,\n    nodeConfig: NodeConfig,\n    sandboxService: SandboxService,\n    dsProcessorService: DsProcessorService,\n    dynamicDsService: DynamicDsService,\n    unfinalizedBlocksService: UnfinalizedBlocksService,\n    @Inject('IProjectService') private projectService: ProjectService,\n  ) {\n    super(\n      apiService,\n      nodeConfig,\n      sandboxService,\n      dsProcessorService,\n      dynamicDsService,\n      unfinalizedBlocksService,\n      FilterTypeMap,\n      ProcessorTypeMap,\n    );\n  }\n\n  async start(): Promise<void> {\n    await this.projectService.init();\n    logger.info('indexer manager started');\n  }\n\n  @profiler(yargsOptions.argv.profiler)\n  async indexBlock(\n    block: EthereumBlockWrapper,\n    dataSources: SubqlEthereumDataSource[],\n  ): Promise<ProcessBlockResponse> {\n    return super.internalIndexBlock(block, dataSources, () =>\n      this.getApi(block),\n    );\n  }\n\n  getBlockHeight(block: EthereumBlockWrapper): number {\n    return block.blockHeight;\n  }\n\n  getBlockHash(block: EthereumBlockWrapper): string {\n    return block.block.hash;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  private async getApi(block: EthereumBlockWrapper): Promise<SafeEthProvider> {\n    return this.apiService.api.getSafeApi(this.getBlockHeight(block));\n  }\n\n  protected async indexBlockData(\n    { block, transactions }: EthereumBlockWrapper,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    await this.indexBlockContent(block, dataSources, getVM);\n\n    for (const tx of transactions) {\n      await this.indexTransaction(tx, dataSources, getVM);\n\n      for (const log of tx.logs ?? []) {\n        await this.indexEvent(log, dataSources, getVM);\n      }\n    }\n  }\n\n  private async indexBlockContent(\n    block: EthereumBlock,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Block, block, ds, getVM);\n    }\n  }\n\n  private async indexTransaction(\n    tx: EthereumTransaction,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Call, tx, ds, getVM);\n    }\n  }\n\n  private async indexEvent(\n    log: EthereumLog,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Event, log, ds, getVM);\n    }\n  }\n\n  protected async prepareFilteredData(\n    kind: EthereumHandlerKind,\n    data: any,\n    ds: SubqlRuntimeDatasource,\n  ): Promise<any> {\n    return DataAbiParser[kind](this.apiService.api)(data, ds);\n  }\n\n  protected baseCustomHandlerFilter(\n    kind: EthereumHandlerKind,\n    data: any,\n    baseFilter: any,\n  ): boolean {\n    switch (kind) {\n      case EthereumHandlerKind.Block:\n        return EthereumBlockWrapped.filterBlocksProcessor(\n          data as EthereumBlock,\n          baseFilter,\n        );\n      case EthereumHandlerKind.Call:\n        return EthereumBlockWrapped.filterTransactionsProcessor(\n          data as EthereumTransaction,\n          baseFilter,\n        );\n      case EthereumHandlerKind.Event:\n        return EthereumBlockWrapped.filterLogsProcessor(\n          data as EthereumLog,\n          baseFilter,\n        );\n      default:\n        throw new Error('Unsupported handler kind');\n    }\n  }\n}\n\ntype ProcessorTypeMap = {\n  [EthereumHandlerKind.Block]: typeof isBlockHandlerProcessor;\n  [EthereumHandlerKind.Event]: typeof isEventHandlerProcessor;\n  [EthereumHandlerKind.Call]: typeof isCallHandlerProcessor;\n};\n\nconst ProcessorTypeMap = {\n  [EthereumHandlerKind.Block]: isBlockHandlerProcessor,\n  [EthereumHandlerKind.Event]: isEventHandlerProcessor,\n  [EthereumHandlerKind.Call]: isCallHandlerProcessor,\n};\n\nconst FilterTypeMap = {\n  [EthereumHandlerKind.Block]: EthereumBlockWrapped.filterBlocksProcessor,\n  [EthereumHandlerKind.Event]: EthereumBlockWrapped.filterLogsProcessor,\n  [EthereumHandlerKind.Call]: EthereumBlockWrapped.filterTransactionsProcessor,\n};\n\nconst DataAbiParser = {\n  [EthereumHandlerKind.Block]: () => (data: EthereumBlock) => data,\n  [EthereumHandlerKind.Event]:\n    (api: EthereumApi) => (data: EthereumLog, ds: SubqlRuntimeDatasource) =>\n      api.parseLog(data, ds),\n  [EthereumHandlerKind.Call]:\n    (api: EthereumApi) =>\n    (data: EthereumTransaction, ds: SubqlRuntimeDatasource) =>\n      api.parseTransaction(data, ds),\n};\n"]}

package/dist/indexer/indexer.module.js

@@ -31,7 +31,6 @@ IndexerModule = __decorate([
             node_core_1.StoreCacheService,
             node_core_1.StoreService,
             node_core_1.ConnectionPoolService,
-            ethereum_1.EthereumApiService,
             {
                 provide: node_core_1.ApiService,
                 useFactory: async (project, connectionPoolService, eventEmitter) => {

package/dist/indexer/indexer.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.module.js","sourceRoot":"","sources":["../../src/indexer/indexer.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,mDAA8C;AAC9C,2CAAwC;AACxC,yDAAsD;AACtD,gDAQ0B;AAE1B,0CAAiD;AAEjD,iEAA4D;AAC5D,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AACvE,4DAAwD;AACxD,gGAA2F;AAwDpF,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IAtDzB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,gCAAc;YACd,6BAAiB;YACjB,wBAAY;YACZ,iCAAqB;YACrB,6BAAkB;YAClB;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EACf,OAAwB,EACxB,qBAAmE,EACnE,YAA2B,EAC3B,EAAE;oBACF,MAAM,UAAU,GAAG,IAAI,6BAAkB,CACvC,OAAO,EACP,qBAAqB,EACrB,YAAY,CACb,CAAC;oBACF,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,iCAAqB,EAAE,6BAAa,CAAC;aACnE;YACD,gCAAc;YACd,yCAAkB;YAClB;gBACE,OAAO,EAAE,qCAAgB;gBACzB,UAAU,EAAE,GAAG,EAAE;oBACf,IAAI,6BAAY,EAAE;wBAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;qBACjD;oBACD,OAAO,IAAI,kCAAsB,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC;gBAC1D,CAAC;aACF;YACD,sBAAU;YACV,sBAAU;YACV;gBACE,OAAO,EAAE,iBAAiB;gBAC1B,QAAQ,EAAE,gCAAc;aACzB;YACD,8BAAa;YACb;gBACE,OAAO,EAAE,oDAAwB;gBACjC,UAAU,EAAE,GAAG,EAAE;oBACf,IAAI,6BAAY,EAAE;wBAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;qBACjD;oBACD,OAAO,IAAI,iEAA8B,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC;gBAClE,CAAC;aACF;SACF;QACD,OAAO,EAAE,CAAC,wBAAY,CAAC;KACxB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { isMainThread } from 'worker_threads';\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  StoreService,\n  PoiService,\n  MmrService,\n  ConnectionPoolService,\n  StoreCacheService,\n  WorkerDynamicDsService,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum';\nimport { EthereumApiConnection } from '../ethereum/api.connection';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\nimport { WorkerService } from './worker/worker.service';\nimport { WorkerUnfinalizedBlocksService } from './worker/worker.unfinalizedBlocks.service';\n\n@Module({\n  providers: [\n    IndexerManager,\n    StoreCacheService,\n    StoreService,\n    ConnectionPoolService,\n    EthereumApiService,\n    {\n      provide: ApiService,\n      useFactory: async (\n        project: SubqueryProject,\n        connectionPoolService: ConnectionPoolService<EthereumApiConnection>,\n        eventEmitter: EventEmitter2,\n      ) => {\n        const apiService = new EthereumApiService(\n          project,\n          connectionPoolService,\n          eventEmitter,\n        );\n        await apiService.init();\n        return apiService;\n      },\n      inject: ['ISubqueryProject', ConnectionPoolService, EventEmitter2],\n    },\n    SandboxService,\n    DsProcessorService,\n    {\n      provide: DynamicDsService,\n      useFactory: () => {\n        if (isMainThread) {\n          throw new Error('Expected to be worker thread');\n        }\n        return new WorkerDynamicDsService((global as any).host);\n      },\n    },\n    PoiService,\n    MmrService,\n    {\n      provide: 'IProjectService',\n      useClass: ProjectService,\n    },\n    WorkerService,\n    {\n      provide: UnfinalizedBlocksService,\n      useFactory: () => {\n        if (isMainThread) {\n          throw new Error('Expected to be worker thread');\n        }\n        return new WorkerUnfinalizedBlocksService((global as any).host);\n      },\n    },\n  ],\n  exports: [StoreService],\n})\nexport class IndexerModule {}\n"]}
+{"version":3,"file":"indexer.module.js","sourceRoot":"","sources":["../../src/indexer/indexer.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,mDAA8C;AAC9C,2CAAwC;AACxC,yDAAsD;AACtD,gDAQ0B;AAE1B,0CAAiD;AAEjD,iEAA4D;AAC5D,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AACvE,4DAAwD;AACxD,gGAA2F;AAuDpF,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IArDzB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,gCAAc;YACd,6BAAiB;YACjB,wBAAY;YACZ,iCAAqB;YACrB;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EACf,OAAwB,EACxB,qBAAmE,EACnE,YAA2B,EAC3B,EAAE;oBACF,MAAM,UAAU,GAAG,IAAI,6BAAkB,CACvC,OAAO,EACP,qBAAqB,EACrB,YAAY,CACb,CAAC;oBACF,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,iCAAqB,EAAE,6BAAa,CAAC;aACnE;YACD,gCAAc;YACd,yCAAkB;YAClB;gBACE,OAAO,EAAE,qCAAgB;gBACzB,UAAU,EAAE,GAAG,EAAE;oBACf,IAAI,6BAAY,EAAE;wBAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;qBACjD;oBACD,OAAO,IAAI,kCAAsB,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC;gBAC1D,CAAC;aACF;YACD,sBAAU;YACV,sBAAU;YACV;gBACE,OAAO,EAAE,iBAAiB;gBAC1B,QAAQ,EAAE,gCAAc;aACzB;YACD,8BAAa;YACb;gBACE,OAAO,EAAE,oDAAwB;gBACjC,UAAU,EAAE,GAAG,EAAE;oBACf,IAAI,6BAAY,EAAE;wBAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;qBACjD;oBACD,OAAO,IAAI,iEAA8B,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC;gBAClE,CAAC;aACF;SACF;QACD,OAAO,EAAE,CAAC,wBAAY,CAAC;KACxB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { isMainThread } from 'worker_threads';\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  StoreService,\n  PoiService,\n  MmrService,\n  ConnectionPoolService,\n  StoreCacheService,\n  WorkerDynamicDsService,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum';\nimport { EthereumApiConnection } from '../ethereum/api.connection';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\nimport { WorkerService } from './worker/worker.service';\nimport { WorkerUnfinalizedBlocksService } from './worker/worker.unfinalizedBlocks.service';\n\n@Module({\n  providers: [\n    IndexerManager,\n    StoreCacheService,\n    StoreService,\n    ConnectionPoolService,\n    {\n      provide: ApiService,\n      useFactory: async (\n        project: SubqueryProject,\n        connectionPoolService: ConnectionPoolService<EthereumApiConnection>,\n        eventEmitter: EventEmitter2,\n      ) => {\n        const apiService = new EthereumApiService(\n          project,\n          connectionPoolService,\n          eventEmitter,\n        );\n        await apiService.init();\n        return apiService;\n      },\n      inject: ['ISubqueryProject', ConnectionPoolService, EventEmitter2],\n    },\n    SandboxService,\n    DsProcessorService,\n    {\n      provide: DynamicDsService,\n      useFactory: () => {\n        if (isMainThread) {\n          throw new Error('Expected to be worker thread');\n        }\n        return new WorkerDynamicDsService((global as any).host);\n      },\n    },\n    PoiService,\n    MmrService,\n    {\n      provide: 'IProjectService',\n      useClass: ProjectService,\n    },\n    WorkerService,\n    {\n      provide: UnfinalizedBlocksService,\n      useFactory: () => {\n        if (isMainThread) {\n          throw new Error('Expected to be worker thread');\n        }\n        return new WorkerUnfinalizedBlocksService((global as any).host);\n      },\n    },\n  ],\n  exports: [StoreService],\n})\nexport class IndexerModule {}\n"]}

package/dist/indexer/project.service.d.ts

@@ -1,43 +1,13 @@
 import { EventEmitter2 } from '@nestjs/event-emitter';
-import { ApiService, IProjectService, NodeConfig, StoreService, PoiService, MmrService } from '@subql/node-core';
+import { PoiService, MmrService, BaseProjectService, StoreService, NodeConfig, ApiService } from '@subql/node-core';
 import { Sequelize } from 'sequelize';
 import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';
 import { DsProcessorService } from './ds-processor.service';
 import { DynamicDsService } from './dynamic-ds.service';
 import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
-export declare class ProjectService implements IProjectService<SubqlProjectDs> {
-    private readonly dsProcessorService;
-    private readonly apiService;
-    private readonly poiService;
-    protected readonly mmrService: MmrService;
-    private readonly sequelize;
-    private readonly project;
-    private readonly storeService;
-    private readonly nodeConfig;
-    private readonly dynamicDsService;
-    private eventEmitter;
-    private unfinalizedBlockService;
-    private _schema;
-    private _startHeight;
-    private _blockOffset;
+export declare class ProjectService extends BaseProjectService<SubqlProjectDs> {
+    protected packageVersion: any;
     constructor(dsProcessorService: DsProcessorService, apiService: ApiService, poiService: PoiService, mmrService: MmrService, sequelize: Sequelize, project: SubqueryProject, storeService: StoreService, nodeConfig: NodeConfig, dynamicDsService: DynamicDsService, eventEmitter: EventEmitter2, unfinalizedBlockService: UnfinalizedBlocksService);
-    get schema(): string;
-    get dataSources(): SubqlProjectDs[];
-    get blockOffset(): number;
-    get startHeight(): number;
-    get isHistorical(): boolean;
-    private getExistingProjectSchema;
-    init(): Promise<void>;
-    private ensureProject;
-    private createProjectSchema;
-    private initHotSchemaReload;
-    private initDbSchema;
-    private ensureMetadata;
-    private getMetadataBlockOffset;
-    private getLastProcessedHeight;
-    private getStartHeight;
-    setBlockOffset(offset: number): Promise<void>;
-    private getStartBlockFromDataSources;
-    reindex(targetBlockHeight: number): Promise<void>;
-    getAllDataSources(blockHeight: number): Promise<SubqlProjectDs[]>;
+    protected generateTimestampReferenceForBlockFilters(ds: SubqlProjectDs[]): Promise<SubqlProjectDs[]>;
+    protected getStartBlockDatasources(): SubqlProjectDs[];
 }

package/dist/indexer/project.service.js

@@ -13,226 +13,29 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 var __param = (this && this.__param) || function (paramIndex, decorator) {
     return function (target, key) { decorator(target, key, paramIndex); }
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ProjectService = void 0;
-const assert_1 = __importDefault(require("assert"));
-const worker_threads_1 = require("worker_threads");
 const common_1 = require("@nestjs/common");
 const event_emitter_1 = require("@nestjs/event-emitter");
 const node_core_1 = require("@subql/node-core");
 const sequelize_1 = require("sequelize");
 const SubqueryProject_1 = require("../configure/SubqueryProject");
-const project_1 = require("../utils/project");
-const reindex_1 = require("../utils/reindex");
 const ds_processor_service_1 = require("./ds-processor.service");
 const dynamic_ds_service_1 = require("./dynamic-ds.service");
 const unfinalizedBlocks_service_1 = require("./unfinalizedBlocks.service");
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const { version: packageVersion } = require('../../package.json');
-const logger = (0, node_core_1.getLogger)('Project');
-let ProjectService = class ProjectService {
+let ProjectService = class ProjectService extends node_core_1.BaseProjectService {
     constructor(dsProcessorService, apiService, poiService, mmrService, sequelize, project, storeService, nodeConfig, dynamicDsService, eventEmitter, unfinalizedBlockService) {
-        this.dsProcessorService = dsProcessorService;
-        this.apiService = apiService;
-        this.poiService = poiService;
-        this.mmrService = mmrService;
-        this.sequelize = sequelize;
-        this.project = project;
-        this.storeService = storeService;
-        this.nodeConfig = nodeConfig;
-        this.dynamicDsService = dynamicDsService;
-        this.eventEmitter = eventEmitter;
-        this.unfinalizedBlockService = unfinalizedBlockService;
+        super(dsProcessorService, apiService, poiService, mmrService, sequelize, project, storeService, nodeConfig, dynamicDsService, eventEmitter, unfinalizedBlockService);
+        this.packageVersion = packageVersion;
     }
-    get schema() {
-        return this._schema;
+    async generateTimestampReferenceForBlockFilters(ds) {
+        return (0, SubqueryProject_1.generateTimestampReferenceForBlockFilters)(ds, this.apiService.api);
     }
-    get dataSources() {
+    getStartBlockDatasources() {
         return this.project.dataSources;
     }
-    get blockOffset() {
-        return this._blockOffset;
-    }
-    get startHeight() {
-        return this._startHeight;
-    }
-    get isHistorical() {
-        return this.storeService.historical;
-    }
-    async getExistingProjectSchema() {
-        return (0, node_core_1.getExistingProjectSchema)(this.nodeConfig, this.sequelize);
-    }
-    async init() {
-        // Used to load assets into DS-processor, has to be done in any thread
-        await this.dsProcessorService.validateProjectCustomDatasources();
-        // Do extra work on main thread to setup stuff
-        this.project.dataSources = await (0, SubqueryProject_1.generateTimestampReferenceForBlockFilters)(this.project.dataSources, this.apiService.api);
-        if (worker_threads_1.isMainThread) {
-            this._schema = await this.ensureProject();
-            await this.initDbSchema();
-            await this.ensureMetadata();
-            this.dynamicDsService.init(this.storeService.storeCache.metadata);
-            await this.initHotSchemaReload();
-            if (this.nodeConfig.proofOfIndex) {
-                const blockOffset = await this.getMetadataBlockOffset();
-                void this.setBlockOffset(Number(blockOffset));
-                await this.poiService.init();
-            }
-            this._startHeight = await this.getStartHeight();
-            if (this.nodeConfig.unfinalizedBlocks && !this.isHistorical) {
-                logger.error('Unfinalized blocks cannot be enabled without historical. You will need to reindex your project to enable historical');
-                process.exit(1);
-            }
-            const reindexedTo = await this.unfinalizedBlockService.init(this.reindex.bind(this));
-            if (reindexedTo !== undefined) {
-                this._startHeight = reindexedTo;
-            }
-            // Flush any pending operations to setup DB
-            await this.storeService.storeCache.flushCache(true);
-        }
-        else {
-            this._schema = await this.getExistingProjectSchema();
-            await this.sequelize.sync();
-            (0, assert_1.default)(this._schema, 'Schema should be created in main thread');
-            await this.initDbSchema();
-            if (this.nodeConfig.proofOfIndex) {
-                await this.poiService.init();
-            }
-        }
-    }
-    async ensureProject() {
-        let schema = await this.getExistingProjectSchema();
-        if (!schema) {
-            schema = await this.createProjectSchema();
-        }
-        this.eventEmitter.emit(node_core_1.IndexerEvent.Ready, {
-            value: true,
-        });
-        return schema;
-    }
-    async createProjectSchema() {
-        const schema = this.nodeConfig.dbSchema;
-        const schemas = await this.sequelize.showAllSchemas(undefined);
-        if (!schemas.includes(schema)) {
-            await this.sequelize.createSchema(`"${schema}"`, undefined);
-        }
-        return schema;
-    }
-    async initHotSchemaReload() {
-        await (0, project_1.initHotSchemaReload)(this.schema, this.storeService);
-    }
-    async initDbSchema() {
-        await (0, project_1.initDbSchema)(this.project, this.schema, this.storeService);
-    }
-    async ensureMetadata() {
-        var _a;
-        const metadata = this.storeService.storeCache.metadata;
-        this.eventEmitter.emit(node_core_1.IndexerEvent.NetworkMetadata, this.apiService.networkMeta);
-        const keys = [
-            'lastProcessedHeight',
-            'blockOffset',
-            'indexerNodeVersion',
-            'chain',
-            'specName',
-            'genesisHash',
-            'startHeight',
-            'processedBlockCount',
-            'lastFinalizedVerifiedHeight',
-            'schemaMigrationCount',
-        ];
-        const existing = await metadata.findMany(keys);
-        const { chain, genesisHash, specName } = this.apiService.networkMeta;
-        if (this.project.runner) {
-            const { node, query } = this.project.runner;
-            metadata.setBulk([
-                { key: 'runnerNode', value: node.name },
-                { key: 'runnerNodeVersion', value: node.version },
-                { key: 'runnerQuery', value: query.name },
-                { key: 'runnerQueryVersion', value: query.version },
-            ]);
-        }
-        if (!existing.genesisHash) {
-            metadata.set('genesisHash', genesisHash);
-        }
-        else {
-            // Check if the configured genesisHash matches the currently stored genesisHash
-            (0, assert_1.default)(
-            // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1
-            ((_a = this.project.network.genesisHash) !== null && _a !== void 0 ? _a : genesisHash) ===
-                existing.genesisHash, 'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean');
-        }
-        if (existing.chain !== chain) {
-            metadata.set('chain', chain);
-        }
-        if (existing.specName !== specName) {
-            metadata.set('specName', specName);
-        }
-        // If project was created before this feature, don't add the key. If it is project created after, add this key.
-        if (!existing.processedBlockCount && !existing.lastProcessedHeight) {
-            metadata.set('processedBlockCount', 0);
-        }
-        if (existing.indexerNodeVersion !== packageVersion) {
-            metadata.set('indexerNodeVersion', packageVersion);
-        }
-        if (!existing.schemaMigrationCount) {
-            metadata.set('schemaMigrationCount', 0);
-        }
-        if (!existing.startHeight) {
-            metadata.set('startHeight', this.getStartBlockFromDataSources());
-        }
-    }
-    async getMetadataBlockOffset() {
-        return this.storeService.storeCache.metadata.find('blockOffset');
-    }
-    async getLastProcessedHeight() {
-        return this.storeService.storeCache.metadata.find('lastProcessedHeight');
-    }
-    async getStartHeight() {
-        let startHeight;
-        const lastProcessedHeight = await this.getLastProcessedHeight();
-        if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {
-            startHeight = Number(lastProcessedHeight) + 1;
-        }
-        else {
-            startHeight = this.getStartBlockFromDataSources();
-        }
-        return startHeight;
-    }
-    async setBlockOffset(offset) {
-        if (this._blockOffset !== undefined ||
-            offset === null ||
-            offset === undefined ||
-            isNaN(offset)) {
-            return;
-        }
-        logger.info(`set blockOffset to ${offset}`);
-        this._blockOffset = offset;
-        return this.mmrService.syncFileBaseFromPoi(offset).catch((err) => {
-            logger.error(err, 'failed to sync poi to mmr');
-            process.exit(1);
-        });
-    }
-    getStartBlockFromDataSources() {
-        const startBlocksList = this.project.dataSources.map((item) => { var _a; return (_a = item.startBlock) !== null && _a !== void 0 ? _a : 1; });
-        if (startBlocksList.length === 0) {
-            logger.error(`Failed to find a valid datasource, Please check your endpoint if specName filter is used.`);
-            process.exit(1);
-        }
-        else {
-            return Math.min(...startBlocksList);
-        }
-    }
-    async reindex(targetBlockHeight) {
-        const lastProcessedHeight = await this.getLastProcessedHeight();
-        return (0, reindex_1.reindex)(this.getStartBlockFromDataSources(), await this.getMetadataBlockOffset(), targetBlockHeight, lastProcessedHeight, this.storeService, this.unfinalizedBlockService, this.dynamicDsService, this.mmrService, this.sequelize);
-    }
-    async getAllDataSources(blockHeight) {
-        const dynamicDs = await this.dynamicDsService.getDynamicDatasources();
-        return [...this.dataSources, ...dynamicDs].filter((ds) => ds.startBlock <= blockHeight);
-    }
 };
 ProjectService = __decorate([
     (0, common_1.Injectable)(),