@subql/node-ethereum 1.10.1-0

package/dist/indexer/project.service.js

@@ -0,0 +1,255 @@
+"use strict";
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProjectService = void 0;
+const assert_1 = __importDefault(require("assert"));
+const worker_threads_1 = require("worker_threads");
+const common_1 = require("@nestjs/common");
+const event_emitter_1 = require("@nestjs/event-emitter");
+const node_core_1 = require("@subql/node-core");
+const sequelize_1 = require("sequelize");
+const SubqueryProject_1 = require("../configure/SubqueryProject");
+const project_1 = require("../utils/project");
+const ds_processor_service_1 = require("./ds-processor.service");
+const dynamic_ds_service_1 = require("./dynamic-ds.service");
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { version: packageVersion } = require('../../package.json');
+const DEFAULT_DB_SCHEMA = 'public';
+const logger = (0, node_core_1.getLogger)('Project');
+let ProjectService = class ProjectService {
+    constructor(dsProcessorService, apiService, poiService, mmrService, sequelize, project, storeService, nodeConfig, dynamicDsService, eventEmitter) {
+        this.dsProcessorService = dsProcessorService;
+        this.apiService = apiService;
+        this.poiService = poiService;
+        this.mmrService = mmrService;
+        this.sequelize = sequelize;
+        this.project = project;
+        this.storeService = storeService;
+        this.nodeConfig = nodeConfig;
+        this.dynamicDsService = dynamicDsService;
+        this.eventEmitter = eventEmitter;
+    }
+    get schema() {
+        return this._schema;
+    }
+    get blockOffset() {
+        return this._blockOffset;
+    }
+    get startHeight() {
+        return this._startHeight;
+    }
+    async getExistingProjectSchema() {
+        return (0, node_core_1.getExistingProjectSchema)(this.nodeConfig, this.sequelize);
+    }
+    async init() {
+        // Used to load assets into DS-processor, has to be done in any thread
+        await this.dsProcessorService.validateProjectCustomDatasources();
+        // Do extra work on main thread to setup stuff
+        if (worker_threads_1.isMainThread) {
+            this._schema = await this.ensureProject();
+            await this.initDbSchema();
+            this.metadataRepo = await this.ensureMetadata();
+            this.dynamicDsService.init(this.metadataRepo);
+            if (this.nodeConfig.proofOfIndex) {
+                const blockOffset = await this.getMetadataBlockOffset();
+                void this.setBlockOffset(Number(blockOffset));
+                await this.poiService.init(this.schema);
+            }
+            this._startHeight = await this.getStartHeight();
+        }
+        else {
+            this.metadataRepo = (0, node_core_1.MetadataFactory)(this.sequelize, this.schema);
+            this.dynamicDsService.init(this.metadataRepo);
+            await this.sequelize.sync();
+            this._schema = await this.getExistingProjectSchema();
+            (0, assert_1.default)(this._schema, 'Schema should be created in main thread');
+            await this.initDbSchema();
+            if (this.nodeConfig.proofOfIndex) {
+                await this.poiService.init(this.schema);
+            }
+        }
+    }
+    async ensureProject() {
+        let schema = await this.getExistingProjectSchema();
+        if (!schema) {
+            schema = await this.createProjectSchema();
+        }
+        this.eventEmitter.emit(node_core_1.IndexerEvent.Ready, {
+            value: true,
+        });
+        return schema;
+    }
+    async createProjectSchema() {
+        let schema;
+        if (this.nodeConfig.localMode) {
+            // create tables in default schema if local mode is enabled
+            schema = DEFAULT_DB_SCHEMA;
+        }
+        else {
+            schema = this.nodeConfig.dbSchema;
+            const schemas = await this.sequelize.showAllSchemas(undefined);
+            if (!schemas.includes(schema)) {
+                await this.sequelize.createSchema(`"${schema}"`, undefined);
+            }
+        }
+        return schema;
+    }
+    async initDbSchema() {
+        await (0, project_1.initDbSchema)(this.project, this.schema, this.storeService);
+    }
+    async ensureMetadata() {
+        var _a;
+        const metadataRepo = (0, node_core_1.MetadataFactory)(this.sequelize, this.schema);
+        this.eventEmitter.emit(node_core_1.IndexerEvent.NetworkMetadata, this.apiService.networkMeta);
+        const keys = [
+            'lastProcessedHeight',
+            'blockOffset',
+            'indexerNodeVersion',
+            'chain',
+            'specName',
+            'genesisHash',
+            'chainId',
+            'processedBlockCount',
+        ];
+        const entries = await metadataRepo.findAll({
+            where: {
+                key: keys,
+            },
+        });
+        const keyValue = entries.reduce((arr, curr) => {
+            arr[curr.key] = curr.value;
+            return arr;
+        }, {});
+        const { chain, genesisHash, specName } = this.apiService.networkMeta;
+        if (this.project.runner) {
+            await Promise.all([
+                metadataRepo.upsert({
+                    key: 'runnerNode',
+                    value: this.project.runner.node.name,
+                }),
+                metadataRepo.upsert({
+                    key: 'runnerNodeVersion',
+                    value: this.project.runner.node.version,
+                }),
+                metadataRepo.upsert({
+                    key: 'runnerQuery',
+                    value: this.project.runner.query.name,
+                }),
+                metadataRepo.upsert({
+                    key: 'runnerQueryVersion',
+                    value: this.project.runner.query.version,
+                }),
+            ]);
+        }
+        if (!keyValue.genesisHash) {
+            await metadataRepo.upsert({ key: 'genesisHash', value: genesisHash });
+        }
+        else {
+            // Check if the configured genesisHash matches the currently stored genesisHash
+            (0, assert_1.default)(
+            // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1
+            ((_a = this.project.network.genesisHash) !== null && _a !== void 0 ? _a : genesisHash) ===
+                keyValue.genesisHash, 'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean');
+        }
+        if (keyValue.chain !== chain) {
+            await metadataRepo.upsert({ key: 'chain', value: chain });
+        }
+        if (keyValue.specName !== specName) {
+            await metadataRepo.upsert({ key: 'specName', value: specName });
+        }
+        // If project was created before this feature, don't add the key. If it is project created after, add this key.
+        if (!keyValue.processedBlockCount && !keyValue.lastProcessedHeight) {
+            await metadataRepo.upsert({ key: 'processedBlockCount', value: 0 });
+        }
+        if (keyValue.indexerNodeVersion !== packageVersion) {
+            await metadataRepo.upsert({
+                key: 'indexerNodeVersion',
+                value: packageVersion,
+            });
+        }
+        return metadataRepo;
+    }
+    async upsertMetadataBlockOffset(height) {
+        await this.metadataRepo.upsert({
+            key: 'blockOffset',
+            value: height,
+        });
+    }
+    async getMetadataBlockOffset() {
+        return (0, node_core_1.getMetaDataInfo)(this.metadataRepo, 'blockOffset');
+    }
+    async getLastProcessedHeight() {
+        return (0, node_core_1.getMetaDataInfo)(this.metadataRepo, 'lastProcessedHeight');
+    }
+    async getStartHeight() {
+        let startHeight;
+        const lastProcessedHeight = await this.getLastProcessedHeight();
+        if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {
+            startHeight = Number(lastProcessedHeight) + 1;
+        }
+        else {
+            startHeight = this.getStartBlockFromDataSources();
+        }
+        return startHeight;
+    }
+    async setBlockOffset(offset) {
+        if (this._blockOffset ||
+            offset === null ||
+            offset === undefined ||
+            isNaN(offset)) {
+            return;
+        }
+        logger.info(`set blockOffset to ${offset}`);
+        this._blockOffset = offset;
+        return this.mmrService
+            .syncFileBaseFromPoi(this.schema, offset)
+            .catch((err) => {
+            logger.error(err, 'failed to sync poi to mmr');
+            process.exit(1);
+        });
+    }
+    async getProcessedBlockCount() {
+        const res = await this.metadataRepo.findOne({
+            where: { key: 'processedBlockCount' },
+        });
+        return res === null || res === void 0 ? void 0 : res.value;
+    }
+    getStartBlockFromDataSources() {
+        const startBlocksList = this.project.dataSources.map((item) => { var _a; return (_a = item.startBlock) !== null && _a !== void 0 ? _a : 1; });
+        if (startBlocksList.length === 0) {
+            logger.error(`Failed to find a valid datasource, Please check your endpoint if specName filter is used.`);
+            process.exit(1);
+        }
+        else {
+            return Math.min(...startBlocksList);
+        }
+    }
+};
+ProjectService = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [ds_processor_service_1.DsProcessorService,
+        node_core_1.ApiService,
+        node_core_1.PoiService,
+        node_core_1.MmrService,
+        sequelize_1.Sequelize,
+        SubqueryProject_1.SubqueryProject,
+        node_core_1.StoreService,
+        node_core_1.NodeConfig,
+        dynamic_ds_service_1.DynamicDsService,
+        event_emitter_1.EventEmitter2])
+], ProjectService);
+exports.ProjectService = ProjectService;
+//# sourceMappingURL=project.service.js.map

package/dist/indexer/project.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,mDAA8C;AAC9C,2CAA4C;AAC5C,yDAAsD;AACtD,gDAY0B;AAC1B,yCAAsC;AACtC,kEAA+D;AAC/D,8CAAgD;AAChD,iEAA4D;AAC5D,6DAAwD;AAExD,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,iBAAiB,GAAG,QAAQ,CAAC;AAEnC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAMzB,YACmB,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACpB,UAAsB,EACxB,SAAoB,EACpB,OAAwB,EACxB,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAC3C,YAA2B;QATlB,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACpB,eAAU,GAAV,UAAU,CAAY;QACxB,cAAS,GAAT,SAAS,CAAW;QACpB,YAAO,GAAP,OAAO,CAAiB;QACxB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAC3C,iBAAY,GAAZ,YAAY,CAAe;IAClC,CAAC;IAEJ,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,sEAAsE;QACtE,MAAM,IAAI,CAAC,kBAAkB,CAAC,gCAAgC,EAAE,CAAC;QACjE,8CAA8C;QAC9C,IAAI,6BAAY,EAAE;YAChB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1B,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAChD,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE9C,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACzC;YAED,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;SACjD;aAAM;YACL,IAAI,CAAC,YAAY,GAAG,IAAA,2BAAe,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE9C,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;YACrD,IAAA,gBAAM,EAAC,IAAI,CAAC,OAAO,EAAE,yCAAyC,CAAC,CAAC;YAChE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACzC;SACF;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,KAAK,EAAE;YACzC,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,MAAc,CAAC;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;YAC7B,2DAA2D;YAC3D,MAAM,GAAG,iBAAiB,CAAC;SAC5B;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAE,OAA+B,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;gBACtD,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;aAC7D;SACF;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;;QAC1B,MAAM,YAAY,GAAG,IAAA,2BAAe,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAElE,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,wBAAY,CAAC,eAAe,EAC5B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QAEF,MAAM,IAAI,GAAG;YACX,qBAAqB;YACrB,aAAa;YACb,oBAAoB;YACpB,OAAO;YACP,UAAU;YACV,aAAa;YACb,SAAS;YACT,qBAAqB;SACb,CAAC;QAEX,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC;YACzC,KAAK,EAAE;gBACL,GAAG,EAAE,IAAI;aACV;SACF,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YAC5C,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;YAC3B,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,EAAiE,CAAC,CAAC;QAEtE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAErE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,OAAO,CAAC,GAAG,CAAC;gBAChB,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,YAAY;oBACjB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI;iBACrC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,mBAAmB;oBACxB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO;iBACxC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,aAAa;oBAClB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;iBACtC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,oBAAoB;oBACzB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO;iBACzC,CAAC;aACH,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;SACvE;aAAM;YACL,+EAA+E;YAC/E,IAAA,gBAAM;YACJ,uHAAuH;YACvH,CAAC,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,mCAAI,WAAW,CAAC;gBAC/C,QAAQ,CAAC,WAAW,EACtB,sJAAsJ,CACvJ,CAAC;SACH;QAED,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE;YAC5B,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;SAC3D;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAClC,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;SACjE;QAED,+GAA+G;QAC/G,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE;YAClE,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,qBAAqB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;SACrE;QAED,IAAI,QAAQ,CAAC,kBAAkB,KAAK,cAAc,EAAE;YAClD,MAAM,YAAY,CAAC,MAAM,CAAC;gBACxB,GAAG,EAAE,oBAAoB;gBACzB,KAAK,EAAE,cAAc;aACtB,CAAC,CAAC;SACJ;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,MAAc;QAC5C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAC7B,GAAG,EAAE,aAAa;YAClB,KAAK,EAAE,MAAM;SACd,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,WAAmB,CAAC;QACxB,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACrE,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;SAC/C;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACnD;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,IACE,IAAI,CAAC,YAAY;YACjB,MAAM,KAAK,IAAI;YACf,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,CAAC,EACb;YACA,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,UAAU;aACnB,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;aACxC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,2BAA2B,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACP,CAAC;IACD,KAAK,CAAC,sBAAsB;QAC1B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;YAC1C,KAAK,EAAE,EAAE,GAAG,EAAE,qBAAqB,EAAE;SACtC,CAAC,CAAC;QAEH,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAA2B,CAAC;IAC1C,CAAC;IAEO,4BAA4B;QAClC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAC/B,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;CACF,CAAA;AA9PY,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAQ4B,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACR,sBAAU;QACb,qBAAS;QACX,iCAAe;QACV,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QAC7B,6BAAa;GAhB1B,cAAc,CA8P1B;AA9PY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  MetadataFactory,\n  MetadataRepo,\n  NodeConfig,\n  IndexerEvent,\n  StoreService,\n  PoiService,\n  MmrService,\n  getLogger,\n  getExistingProjectSchema,\n  getMetaDataInfo,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { initDbSchema } from '../utils/project';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst DEFAULT_DB_SCHEMA = 'public';\n\nconst logger = getLogger('Project');\n\n@Injectable()\nexport class ProjectService {\n  private _schema: string;\n  private metadataRepo: MetadataRepo;\n  private _startHeight: number;\n  private _blockOffset: number;\n\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    private readonly apiService: ApiService,\n    private readonly poiService: PoiService,\n    protected readonly mmrService: MmrService,\n    private readonly sequelize: Sequelize,\n    private readonly project: SubqueryProject,\n    private readonly storeService: StoreService,\n    private readonly nodeConfig: NodeConfig,\n    private readonly dynamicDsService: DynamicDsService,\n    private eventEmitter: EventEmitter2,\n  ) {}\n\n  get schema(): string {\n    return this._schema;\n  }\n\n  get blockOffset(): number {\n    return this._blockOffset;\n  }\n\n  get startHeight(): number {\n    return this._startHeight;\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  async init(): Promise<void> {\n    // Used to load assets into DS-processor, has to be done in any thread\n    await this.dsProcessorService.validateProjectCustomDatasources();\n    // Do extra work on main thread to setup stuff\n    if (isMainThread) {\n      this._schema = await this.ensureProject();\n      await this.initDbSchema();\n      this.metadataRepo = await this.ensureMetadata();\n      this.dynamicDsService.init(this.metadataRepo);\n\n      if (this.nodeConfig.proofOfIndex) {\n        const blockOffset = await this.getMetadataBlockOffset();\n        void this.setBlockOffset(Number(blockOffset));\n        await this.poiService.init(this.schema);\n      }\n\n      this._startHeight = await this.getStartHeight();\n    } else {\n      this.metadataRepo = MetadataFactory(this.sequelize, this.schema);\n\n      this.dynamicDsService.init(this.metadataRepo);\n\n      await this.sequelize.sync();\n\n      this._schema = await this.getExistingProjectSchema();\n      assert(this._schema, 'Schema should be created in main thread');\n      await this.initDbSchema();\n\n      if (this.nodeConfig.proofOfIndex) {\n        await this.poiService.init(this.schema);\n      }\n    }\n  }\n\n  private async ensureProject(): Promise<string> {\n    let schema = await this.getExistingProjectSchema();\n    if (!schema) {\n      schema = await this.createProjectSchema();\n    }\n    this.eventEmitter.emit(IndexerEvent.Ready, {\n      value: true,\n    });\n\n    return schema;\n  }\n\n  private async createProjectSchema(): Promise<string> {\n    let schema: string;\n    if (this.nodeConfig.localMode) {\n      // create tables in default schema if local mode is enabled\n      schema = DEFAULT_DB_SCHEMA;\n    } else {\n      schema = this.nodeConfig.dbSchema;\n      const schemas = await this.sequelize.showAllSchemas(undefined);\n      if (!(schemas as unknown as string[]).includes(schema)) {\n        await this.sequelize.createSchema(`\"${schema}\"`, undefined);\n      }\n    }\n\n    return schema;\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private async ensureMetadata(): Promise<MetadataRepo> {\n    const metadataRepo = MetadataFactory(this.sequelize, this.schema);\n\n    this.eventEmitter.emit(\n      IndexerEvent.NetworkMetadata,\n      this.apiService.networkMeta,\n    );\n\n    const keys = [\n      'lastProcessedHeight',\n      'blockOffset',\n      'indexerNodeVersion',\n      'chain',\n      'specName',\n      'genesisHash',\n      'chainId',\n      'processedBlockCount',\n    ] as const;\n\n    const entries = await metadataRepo.findAll({\n      where: {\n        key: keys,\n      },\n    });\n\n    const keyValue = entries.reduce((arr, curr) => {\n      arr[curr.key] = curr.value;\n      return arr;\n    }, {} as { [key in typeof keys[number]]: string | boolean | number });\n\n    const { chain, genesisHash, specName } = this.apiService.networkMeta;\n\n    if (this.project.runner) {\n      await Promise.all([\n        metadataRepo.upsert({\n          key: 'runnerNode',\n          value: this.project.runner.node.name,\n        }),\n        metadataRepo.upsert({\n          key: 'runnerNodeVersion',\n          value: this.project.runner.node.version,\n        }),\n        metadataRepo.upsert({\n          key: 'runnerQuery',\n          value: this.project.runner.query.name,\n        }),\n        metadataRepo.upsert({\n          key: 'runnerQueryVersion',\n          value: this.project.runner.query.version,\n        }),\n      ]);\n    }\n    if (!keyValue.genesisHash) {\n      await metadataRepo.upsert({ key: 'genesisHash', value: genesisHash });\n    } else {\n      // Check if the configured genesisHash matches the currently stored genesisHash\n      assert(\n        // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1\n        (this.project.network.genesisHash ?? genesisHash) ===\n          keyValue.genesisHash,\n        'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean',\n      );\n    }\n\n    if (keyValue.chain !== chain) {\n      await metadataRepo.upsert({ key: 'chain', value: chain });\n    }\n\n    if (keyValue.specName !== specName) {\n      await metadataRepo.upsert({ key: 'specName', value: specName });\n    }\n\n    // If project was created before this feature, don't add the key. If it is project created after, add this key.\n    if (!keyValue.processedBlockCount && !keyValue.lastProcessedHeight) {\n      await metadataRepo.upsert({ key: 'processedBlockCount', value: 0 });\n    }\n\n    if (keyValue.indexerNodeVersion !== packageVersion) {\n      await metadataRepo.upsert({\n        key: 'indexerNodeVersion',\n        value: packageVersion,\n      });\n    }\n\n    return metadataRepo;\n  }\n\n  async upsertMetadataBlockOffset(height: number): Promise<void> {\n    await this.metadataRepo.upsert({\n      key: 'blockOffset',\n      value: height,\n    });\n  }\n\n  async getMetadataBlockOffset(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'blockOffset');\n  }\n\n  async getLastProcessedHeight(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'lastProcessedHeight');\n  }\n\n  private async getStartHeight(): Promise<number> {\n    let startHeight: number;\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n    if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {\n      startHeight = Number(lastProcessedHeight) + 1;\n    } else {\n      startHeight = this.getStartBlockFromDataSources();\n    }\n\n    return startHeight;\n  }\n\n  async setBlockOffset(offset: number): Promise<void> {\n    if (\n      this._blockOffset ||\n      offset === null ||\n      offset === undefined ||\n      isNaN(offset)\n    ) {\n      return;\n    }\n    logger.info(`set blockOffset to ${offset}`);\n    this._blockOffset = offset;\n    return this.mmrService\n      .syncFileBaseFromPoi(this.schema, offset)\n      .catch((err) => {\n        logger.error(err, 'failed to sync poi to mmr');\n        process.exit(1);\n      });\n  }\n  async getProcessedBlockCount(): Promise<number> {\n    const res = await this.metadataRepo.findOne({\n      where: { key: 'processedBlockCount' },\n    });\n\n    return res?.value as number | undefined;\n  }\n\n  private getStartBlockFromDataSources() {\n    const startBlocksList = this.project.dataSources.map(\n      (item) => item.startBlock ?? 1,\n    );\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n}\n"]}

package/dist/indexer/sandbox.service.d.ts

@@ -0,0 +1,12 @@
+import { NodeConfig, StoreService, IndexerSandbox } from '@subql/node-core';
+import { ApiWrapper, EthereumBlockWrapper } from '@subql/types-ethereum';
+import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';
+export declare class SandboxService {
+    private readonly storeService;
+    private readonly nodeConfig;
+    private readonly project;
+    private processorCache;
+    constructor(storeService: StoreService, nodeConfig: NodeConfig, project: SubqueryProject);
+    getDsProcessorWrapper(ds: SubqlProjectDs, api: ApiWrapper, blockContent: EthereumBlockWrapper): IndexerSandbox;
+    private getDataSourceEntry;
+}

package/dist/indexer/sandbox.service.js

@@ -0,0 +1,58 @@
+"use strict";
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SandboxService = void 0;
+const common_1 = require("@nestjs/common");
+const common_ethereum_1 = require("@subql/common-ethereum");
+const node_core_1 = require("@subql/node-core");
+const SubqueryProject_1 = require("../configure/SubqueryProject");
+const project_1 = require("../utils/project");
+let SandboxService = class SandboxService {
+    constructor(storeService, nodeConfig, project) {
+        this.storeService = storeService;
+        this.nodeConfig = nodeConfig;
+        this.project = project;
+        this.processorCache = {};
+    }
+    getDsProcessorWrapper(ds, api, blockContent) {
+        const entry = this.getDataSourceEntry(ds);
+        let processor = this.processorCache[entry];
+        if (!processor) {
+            processor = new node_core_1.IndexerSandbox({
+                store: this.storeService.getStore(),
+                root: this.project.root,
+                script: ds.mapping.entryScript,
+                entry,
+            }, this.nodeConfig);
+            this.processorCache[entry] = processor;
+        }
+        api.freezeApi(processor, blockContent);
+        return processor;
+    }
+    getDataSourceEntry(ds) {
+        if ((0, common_ethereum_1.isDatasourceV0_2_0)(ds)) {
+            return ds.mapping.file;
+        }
+        else {
+            return (0, project_1.getProjectEntry)(this.project.root);
+        }
+    }
+};
+SandboxService = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [node_core_1.StoreService,
+        node_core_1.NodeConfig,
+        SubqueryProject_1.SubqueryProject])
+], SandboxService);
+exports.SandboxService = SandboxService;
+//# sourceMappingURL=sandbox.service.js.map

package/dist/indexer/sandbox.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sandbox.service.js","sourceRoot":"","sources":["../../src/indexer/sandbox.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,4DAGgC;AAChC,gDAA4E;AAE5E,kEAA+E;AAC/E,8CAAmD;AAG5C,IAAM,cAAc,GAApB,MAAM,cAAc;IAGzB,YACmB,YAA0B,EAC1B,UAAsB,EACtB,OAAwB;QAFxB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,YAAO,GAAP,OAAO,CAAiB;QALnC,mBAAc,GAAmC,EAAE,CAAC;IAMzD,CAAC;IAEJ,qBAAqB,CACnB,EAAkB,EAClB,GAAe,EACf,YAAkC;QAElC,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,SAAS,EAAE;YACd,SAAS,GAAG,IAAI,0BAAc,CAC5B;gBACE,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE;gBACnC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;gBACvB,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW;gBAC9B,KAAK;aACN,EACD,IAAI,CAAC,UAAU,CAChB,CAAC;YACF,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;SACxC;QACD,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACvC,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,kBAAkB,CAAC,EAA2B;QACpD,IAAI,IAAA,oCAAkB,EAAC,EAAE,CAAC,EAAE;YAC1B,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;SACxB;aAAM;YACL,OAAO,IAAA,yBAAe,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC3C;IACH,CAAC;CACF,CAAA;AAvCY,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAKsB,wBAAY;QACd,sBAAU;QACb,iCAAe;GANhC,cAAc,CAuC1B;AAvCY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport {\n  isDatasourceV0_2_0,\n  SubqlEthereumDataSource,\n} from '@subql/common-ethereum';\nimport { NodeConfig, StoreService, IndexerSandbox } from '@subql/node-core';\nimport { ApiWrapper, EthereumBlockWrapper } from '@subql/types-ethereum';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { getProjectEntry } from '../utils/project';\n\n@Injectable()\nexport class SandboxService {\n  private processorCache: Record<string, IndexerSandbox> = {};\n\n  constructor(\n    private readonly storeService: StoreService,\n    private readonly nodeConfig: NodeConfig,\n    private readonly project: SubqueryProject,\n  ) {}\n\n  getDsProcessorWrapper(\n    ds: SubqlProjectDs,\n    api: ApiWrapper,\n    blockContent: EthereumBlockWrapper,\n  ): IndexerSandbox {\n    const entry = this.getDataSourceEntry(ds);\n    let processor = this.processorCache[entry];\n    if (!processor) {\n      processor = new IndexerSandbox(\n        {\n          store: this.storeService.getStore(),\n          root: this.project.root,\n          script: ds.mapping.entryScript,\n          entry,\n        },\n        this.nodeConfig,\n      );\n      this.processorCache[entry] = processor;\n    }\n    api.freezeApi(processor, blockContent);\n    return processor;\n  }\n\n  private getDataSourceEntry(ds: SubqlEthereumDataSource): string {\n    if (isDatasourceV0_2_0(ds)) {\n      return ds.mapping.file;\n    } else {\n      return getProjectEntry(this.project.root);\n    }\n  }\n}\n"]}

package/dist/indexer/types.d.ts

@@ -0,0 +1,10 @@
+import { Entity } from '@subql/types-ethereum';
+export declare enum OperationType {
+    Set = "Set",
+    Remove = "Remove"
+}
+export declare type OperationEntity = {
+    operation: OperationType;
+    entityType: string;
+    data: Entity | string;
+};

package/dist/indexer/types.js

@@ -0,0 +1,11 @@
+"use strict";
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OperationType = void 0;
+var OperationType;
+(function (OperationType) {
+    OperationType["Set"] = "Set";
+    OperationType["Remove"] = "Remove";
+})(OperationType = exports.OperationType || (exports.OperationType = {}));
+//# sourceMappingURL=types.js.map

package/dist/indexer/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/indexer/types.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAItC,IAAY,aAGX;AAHD,WAAY,aAAa;IACvB,4BAAW,CAAA;IACX,kCAAiB,CAAA;AACnB,CAAC,EAHW,aAAa,GAAb,qBAAa,KAAb,qBAAa,QAGxB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Entity } from '@subql/types-ethereum';\n\nexport enum OperationType {\n  Set = 'Set',\n  Remove = 'Remove',\n}\n\nexport type OperationEntity = {\n  operation: OperationType;\n  entityType: string;\n  data: Entity | string;\n};\n"]}

package/dist/indexer/worker/block-dispatcher.service.d.ts

@@ -0,0 +1,69 @@
+import { OnApplicationShutdown } from '@nestjs/common';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { ApiService, NodeConfig } from '@subql/node-core';
+import { IndexerManager } from '../indexer.manager';
+import { ProjectService } from '../project.service';
+export interface IBlockDispatcher {
+    init(onDynamicDsCreated: (height: number) => Promise<void>): Promise<void>;
+    enqueueBlocks(heights: number[]): void;
+    queueSize: number;
+    freeSize: number;
+    latestBufferedHeight: number | undefined;
+    flushQueue(height: number): void;
+}
+/**
+ * @description Intended to behave the same as WorkerBlockDispatcherService but doesn't use worker threads or any parallel processing
+ */
+export declare class BlockDispatcherService implements IBlockDispatcher, OnApplicationShutdown {
+    private apiService;
+    private nodeConfig;
+    private indexerManager;
+    private eventEmitter;
+    private projectService;
+    private fetchQueue;
+    private processQueue;
+    private fetching;
+    private isShutdown;
+    private onDynamicDsCreated;
+    private _latestBufferedHeight;
+    private _processedBlockCount;
+    private fetchBlocksBatches;
+    private latestProcessedHeight;
+    constructor(apiService: ApiService, nodeConfig: NodeConfig, indexerManager: IndexerManager, eventEmitter: EventEmitter2, projectService: ProjectService);
+    init(onDynamicDsCreated: (height: number) => Promise<void>): Promise<void>;
+    onApplicationShutdown(): void;
+    enqueueBlocks(heights: number[]): void;
+    flushQueue(height: number): void;
+    private setProcessedBlockCount;
+    private fetchBlocksFromQueue;
+    get queueSize(): number;
+    get freeSize(): number;
+    get latestBufferedHeight(): number;
+    set latestBufferedHeight(height: number);
+}
+export declare class WorkerBlockDispatcherService implements IBlockDispatcher, OnApplicationShutdown {
+    private nodeConfig;
+    private eventEmitter;
+    private projectService;
+    private workers;
+    private numWorkers;
+    private onDynamicDsCreated;
+    private taskCounter;
+    private isShutdown;
+    private queue;
+    private _latestBufferedHeight;
+    private _processedBlockCount;
+    constructor(nodeConfig: NodeConfig, eventEmitter: EventEmitter2, projectService: ProjectService);
+    init(onDynamicDsCreated: (height: number) => Promise<void>): Promise<void>;
+    private setProcessedBlockCount;
+    onApplicationShutdown(): Promise<void>;
+    enqueueBlocks(heights: number[]): void;
+    flushQueue(height: number): void;
+    private enqueueBlock;
+    sampleWorkerStatus(): Promise<void>;
+    get queueSize(): number;
+    get freeSize(): number;
+    get latestBufferedHeight(): number;
+    set latestBufferedHeight(height: number);
+    private getNextWorkerIndex;
+}