@subql/node-ethereum 2.0.1 → 2.0.2-1

package/dist/indexer/project.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,mDAA8C;AAC9C,2CAAoD;AACpD,yDAAsD;AACtD,gDAU0B;AAC1B,yCAAsC;AACtC,kEAIsC;AACtC,8CAAqE;AACrE,8CAA2C;AAC3C,iEAA4D;AAC5D,6DAAwD;AACxD,2EAAuE;AAEvE,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAKzB,YACmB,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACpB,UAAsB,EACxB,SAAoB,EACQ,OAAwB,EACpD,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAC3C,YAA2B,EAC3B,uBAAiD;QAVxC,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACpB,eAAU,GAAV,UAAU,CAAY;QACxB,cAAS,GAAT,SAAS,CAAW;QACQ,YAAO,GAAP,OAAO,CAAiB;QACpD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAC3C,iBAAY,GAAZ,YAAY,CAAe;QAC3B,4BAAuB,GAAvB,uBAAuB,CAA0B;IACxD,CAAC;IAEJ,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;IACtC,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,sEAAsE;QACtE,MAAM,IAAI,CAAC,kBAAkB,CAAC,gCAAgC,EAAE,CAAC;QACjE,8CAA8C;QAC9C,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,IAAA,2DAAyC,EACxE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,6BAAY,EAAE;YAChB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAElE,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEjC,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;YAED,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAEhD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;gBAC3D,MAAM,CAAC,KAAK,CACV,qHAAqH,CACtH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;YAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CACzD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CACxB,CAAC;YAEF,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC7B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;aACjC;YAED,2CAA2C;YAC3C,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;SACrD;aAAM;YACL,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAErD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAA,gBAAM,EAAC,IAAI,CAAC,OAAO,EAAE,yCAAyC,CAAC,CAAC;YAChE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;SACF;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,KAAK,EAAE;YACzC,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QACxC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAC/D,IAAI,CAAE,OAA+B,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YACtD,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;SAC7D;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,IAAA,6BAAmB,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5D,CAAC;IACO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAEvD,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,wBAAY,CAAC,eAAe,EAC5B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QAEF,MAAM,IAAI,GAAG;YACX,qBAAqB;YACrB,aAAa;YACb,oBAAoB;YACpB,OAAO;YACP,UAAU;YACV,aAAa;YACb,aAAa;YACb,qBAAqB;YACrB,6BAA6B;YAC7B,sBAAsB;SACd,CAAC;QAEX,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE/C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAErE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE5C,QAAQ,CAAC,OAAO,CAAC;gBACf,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBACvC,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;gBACjD,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;gBACzC,EAAE,GAAG,EAAE,oBAAoB,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE;aACpD,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;SAC1C;aAAM;YACL,+EAA+E;YAC/E,IAAA,gBAAM;YACJ,uHAAuH;YACvH,CAAC,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,mCAAI,WAAW,CAAC;gBAC/C,QAAQ,CAAC,WAAW,EACtB,sJAAsJ,CACvJ,CAAC;SACH;QACD,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE;YAC5B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;SAC9B;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;SACpC;QAED,+GAA+G;QAC/G,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE;YAClE,QAAQ,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;SACxC;QAED,IAAI,QAAQ,CAAC,kBAAkB,KAAK,cAAc,EAAE;YAClD,QAAQ,CAAC,GAAG,CAAC,oBAAoB,EAAE,cAAc,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC;SACzC;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,4BAA4B,EAAE,CAAC,CAAC;SAClE;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC3E,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,WAAmB,CAAC;QACxB,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACrE,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;SAC/C;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACnD;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,IACE,IAAI,CAAC,YAAY,KAAK,SAAS;YAC/B,MAAM,KAAK,IAAI;YACf,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,CAAC,EACb;YACA,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/D,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,2BAA2B,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,4BAA4B;QAClC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAC/B,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEhE,OAAO,IAAA,iBAAO,EACZ,IAAI,CAAC,4BAA4B,EAAE,EACnC,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,uBAAuB,EAC5B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CAEf,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACzC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;QAEtE,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,CAAC,MAAM,CAC/C,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,WAAW,CACrC,CAAC;IACJ,CAAC;CACF,CAAA;AA9QY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAYR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCALU,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACR,sBAAU;QACb,qBAAS;QACiB,iCAAe;QACtC,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QAC7B,6BAAa;QACF,oDAAwB;GAhBhD,cAAc,CA8Q1B;AA9QY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  IProjectService,\n  NodeConfig,\n  IndexerEvent,\n  StoreService,\n  PoiService,\n  MmrService,\n  getLogger,\n  getExistingProjectSchema,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport {\n  generateTimestampReferenceForBlockFilters,\n  SubqlProjectDs,\n  SubqueryProject,\n} from '../configure/SubqueryProject';\nimport { initDbSchema, initHotSchemaReload } from '../utils/project';\nimport { reindex } from '../utils/reindex';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst logger = getLogger('Project');\n\n@Injectable()\nexport class ProjectService implements IProjectService<SubqlProjectDs> {\n  private _schema: string;\n  private _startHeight: number;\n  private _blockOffset: number;\n\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    private readonly apiService: ApiService,\n    private readonly poiService: PoiService,\n    protected readonly mmrService: MmrService,\n    private readonly sequelize: Sequelize,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly storeService: StoreService,\n    private readonly nodeConfig: NodeConfig,\n    private readonly dynamicDsService: DynamicDsService,\n    private eventEmitter: EventEmitter2,\n    private unfinalizedBlockService: UnfinalizedBlocksService,\n  ) {}\n\n  get schema(): string {\n    return this._schema;\n  }\n\n  get dataSources(): SubqlProjectDs[] {\n    return this.project.dataSources;\n  }\n\n  get blockOffset(): number {\n    return this._blockOffset;\n  }\n\n  get startHeight(): number {\n    return this._startHeight;\n  }\n\n  get isHistorical(): boolean {\n    return this.storeService.historical;\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  async init(): Promise<void> {\n    // Used to load assets into DS-processor, has to be done in any thread\n    await this.dsProcessorService.validateProjectCustomDatasources();\n    // Do extra work on main thread to setup stuff\n    this.project.dataSources = await generateTimestampReferenceForBlockFilters(\n      this.project.dataSources,\n      this.apiService.api,\n    );\n    if (isMainThread) {\n      this._schema = await this.ensureProject();\n      await this.initDbSchema();\n      await this.ensureMetadata();\n      this.dynamicDsService.init(this.storeService.storeCache.metadata);\n\n      await this.initHotSchemaReload();\n\n      if (this.nodeConfig.proofOfIndex) {\n        const blockOffset = await this.getMetadataBlockOffset();\n        void this.setBlockOffset(Number(blockOffset));\n        await this.poiService.init();\n      }\n\n      this._startHeight = await this.getStartHeight();\n\n      if (this.nodeConfig.unfinalizedBlocks && !this.isHistorical) {\n        logger.error(\n          'Unfinalized blocks cannot be enabled without historical. You will need to reindex your project to enable historical',\n        );\n        process.exit(1);\n      }\n\n      const reindexedTo = await this.unfinalizedBlockService.init(\n        this.reindex.bind(this),\n      );\n\n      if (reindexedTo !== undefined) {\n        this._startHeight = reindexedTo;\n      }\n\n      // Flush any pending operations to setup DB\n      await this.storeService.storeCache.flushCache(true);\n    } else {\n      this._schema = await this.getExistingProjectSchema();\n\n      await this.sequelize.sync();\n\n      assert(this._schema, 'Schema should be created in main thread');\n      await this.initDbSchema();\n\n      if (this.nodeConfig.proofOfIndex) {\n        await this.poiService.init();\n      }\n    }\n  }\n\n  private async ensureProject(): Promise<string> {\n    let schema = await this.getExistingProjectSchema();\n    if (!schema) {\n      schema = await this.createProjectSchema();\n    }\n    this.eventEmitter.emit(IndexerEvent.Ready, {\n      value: true,\n    });\n\n    return schema;\n  }\n\n  private async createProjectSchema(): Promise<string> {\n    const schema = this.nodeConfig.dbSchema;\n    const schemas = await this.sequelize.showAllSchemas(undefined);\n    if (!(schemas as unknown as string[]).includes(schema)) {\n      await this.sequelize.createSchema(`\"${schema}\"`, undefined);\n    }\n    return schema;\n  }\n\n  private async initHotSchemaReload(): Promise<void> {\n    await initHotSchemaReload(this.schema, this.storeService);\n  }\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private async ensureMetadata(): Promise<void> {\n    const metadata = this.storeService.storeCache.metadata;\n\n    this.eventEmitter.emit(\n      IndexerEvent.NetworkMetadata,\n      this.apiService.networkMeta,\n    );\n\n    const keys = [\n      'lastProcessedHeight',\n      'blockOffset',\n      'indexerNodeVersion',\n      'chain',\n      'specName',\n      'genesisHash',\n      'startHeight',\n      'processedBlockCount',\n      'lastFinalizedVerifiedHeight',\n      'schemaMigrationCount',\n    ] as const;\n\n    const existing = await metadata.findMany(keys);\n\n    const { chain, genesisHash, specName } = this.apiService.networkMeta;\n\n    if (this.project.runner) {\n      const { node, query } = this.project.runner;\n\n      metadata.setBulk([\n        { key: 'runnerNode', value: node.name },\n        { key: 'runnerNodeVersion', value: node.version },\n        { key: 'runnerQuery', value: query.name },\n        { key: 'runnerQueryVersion', value: query.version },\n      ]);\n    }\n    if (!existing.genesisHash) {\n      metadata.set('genesisHash', genesisHash);\n    } else {\n      // Check if the configured genesisHash matches the currently stored genesisHash\n      assert(\n        // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1\n        (this.project.network.genesisHash ?? genesisHash) ===\n          existing.genesisHash,\n        'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean',\n      );\n    }\n    if (existing.chain !== chain) {\n      metadata.set('chain', chain);\n    }\n\n    if (existing.specName !== specName) {\n      metadata.set('specName', specName);\n    }\n\n    // If project was created before this feature, don't add the key. If it is project created after, add this key.\n    if (!existing.processedBlockCount && !existing.lastProcessedHeight) {\n      metadata.set('processedBlockCount', 0);\n    }\n\n    if (existing.indexerNodeVersion !== packageVersion) {\n      metadata.set('indexerNodeVersion', packageVersion);\n    }\n    if (!existing.schemaMigrationCount) {\n      metadata.set('schemaMigrationCount', 0);\n    }\n    if (!existing.startHeight) {\n      metadata.set('startHeight', this.getStartBlockFromDataSources());\n    }\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return this.storeService.storeCache.metadata.find('blockOffset');\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return this.storeService.storeCache.metadata.find('lastProcessedHeight');\n  }\n\n  private async getStartHeight(): Promise<number> {\n    let startHeight: number;\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n    if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {\n      startHeight = Number(lastProcessedHeight) + 1;\n    } else {\n      startHeight = this.getStartBlockFromDataSources();\n    }\n    return startHeight;\n  }\n\n  async setBlockOffset(offset: number): Promise<void> {\n    if (\n      this._blockOffset !== undefined ||\n      offset === null ||\n      offset === undefined ||\n      isNaN(offset)\n    ) {\n      return;\n    }\n    logger.info(`set blockOffset to ${offset}`);\n    this._blockOffset = offset;\n    return this.mmrService.syncFileBaseFromPoi(offset).catch((err) => {\n      logger.error(err, 'failed to sync poi to mmr');\n      process.exit(1);\n    });\n  }\n\n  private getStartBlockFromDataSources() {\n    const startBlocksList = this.project.dataSources.map(\n      (item) => item.startBlock ?? 1,\n    );\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n\n    return reindex(\n      this.getStartBlockFromDataSources(),\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.unfinalizedBlockService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      /* Not providing force clean service, it should never be needed */\n    );\n  }\n\n  async getAllDataSources(blockHeight: number): Promise<SubqlProjectDs[]> {\n    const dynamicDs = await this.dynamicDsService.getDynamicDatasources();\n\n    return [...this.dataSources, ...dynamicDs].filter(\n      (ds) => ds.startBlock <= blockHeight,\n    );\n  }\n}\n"]}
+{"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,yDAAsD;AACtD,gDAO0B;AAC1B,yCAAsC;AACtC,kEAIsC;AACtC,iEAA4D;AAC5D,6DAAwD;AACxD,2EAAuE;AAEvE,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAG3D,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,8BAAkC;IAGpE,YACE,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACtB,UAAsB,EACtB,SAAoB,EACQ,OAAwB,EACpD,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAClC,YAA2B,EAC3B,uBAAiD;QAEjD,KAAK,CACH,kBAAkB,EAClB,UAAU,EACV,UAAU,EACV,UAAU,EACV,SAAS,EACT,OAAO,EACP,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,uBAAuB,CACxB,CAAC;QA3BM,mBAAc,GAAG,cAAc,CAAC;IA4B1C,CAAC;IAES,KAAK,CAAC,yCAAyC,CACvD,EAAoB;QAEpB,OAAO,IAAA,2DAAyC,EAAC,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5E,CAAC;IAES,wBAAwB;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;CACF,CAAA;AAxCY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCALP,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACV,sBAAU;QACX,qBAAS;QACiB,iCAAe;QACtC,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QACpB,6BAAa;QACF,oDAAwB;GAdxC,cAAc,CAwC1B;AAxCY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  PoiService,\n  MmrService,\n  BaseProjectService,\n  StoreService,\n  NodeConfig,\n  ApiService,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport {\n  generateTimestampReferenceForBlockFilters,\n  SubqlProjectDs,\n  SubqueryProject,\n} from '../configure/SubqueryProject';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\n@Injectable()\nexport class ProjectService extends BaseProjectService<SubqlProjectDs> {\n  protected packageVersion = packageVersion;\n\n  constructor(\n    dsProcessorService: DsProcessorService,\n    apiService: ApiService,\n    poiService: PoiService,\n    mmrService: MmrService,\n    sequelize: Sequelize,\n    @Inject('ISubqueryProject') project: SubqueryProject,\n    storeService: StoreService,\n    nodeConfig: NodeConfig,\n    dynamicDsService: DynamicDsService,\n    eventEmitter: EventEmitter2,\n    unfinalizedBlockService: UnfinalizedBlocksService,\n  ) {\n    super(\n      dsProcessorService,\n      apiService,\n      poiService,\n      mmrService,\n      sequelize,\n      project,\n      storeService,\n      nodeConfig,\n      dynamicDsService,\n      eventEmitter,\n      unfinalizedBlockService,\n    );\n  }\n\n  protected async generateTimestampReferenceForBlockFilters(\n    ds: SubqlProjectDs[],\n  ): Promise<SubqlProjectDs[]> {\n    return generateTimestampReferenceForBlockFilters(ds, this.apiService.api);\n  }\n\n  protected getStartBlockDatasources(): SubqlProjectDs[] {\n    return this.project.dataSources;\n  }\n}\n"]}

package/dist/indexer/sandbox.service.d.ts

@@ -1,12 +1,13 @@
-import { NodeConfig, StoreService, IndexerSandbox } from '@subql/node-core';
-import { ApiWrapper, EthereumBlockWrapper } from '@subql/types-ethereum';
-import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';
+import { BaseDataSource } from '@subql/common';
+import { NodeConfig, StoreService, IndexerSandbox, ISubqueryProject, ApiService } from '@subql/node-core';
+import SafeEthProvider from '../ethereum/safe-api';
 export declare class SandboxService {
+    private readonly apiService;
     private readonly storeService;
     private readonly nodeConfig;
     private readonly project;
     private processorCache;
-    constructor(storeService: StoreService, nodeConfig: NodeConfig, project: SubqueryProject);
-    getDsProcessorWrapper(ds: SubqlProjectDs, api: ApiWrapper, blockContent: EthereumBlockWrapper): IndexerSandbox;
+    constructor(apiService: ApiService, storeService: StoreService, nodeConfig: NodeConfig, project: ISubqueryProject);
+    getDsProcessor(ds: BaseDataSource, api: SafeEthProvider): IndexerSandbox;
     private getDataSourceEntry;
 }

package/dist/indexer/sandbox.service.js

@@ -17,18 +17,17 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.SandboxService = void 0;
 const worker_threads_1 = require("worker_threads");
 const common_1 = require("@nestjs/common");
-const common_ethereum_1 = require("@subql/common-ethereum");
 const node_core_1 = require("@subql/node-core");
-const SubqueryProject_1 = require("../configure/SubqueryProject");
-const project_1 = require("../utils/project");
+/* It would be nice to move this to node core but need to find a way to inject other things into the sandbox */
 let SandboxService = class SandboxService {
-    constructor(storeService, nodeConfig, project) {
+    constructor(apiService, storeService, nodeConfig, project) {
+        this.apiService = apiService;
         this.storeService = storeService;
         this.nodeConfig = nodeConfig;
         this.project = project;
         this.processorCache = {};
     }
-    getDsProcessorWrapper(ds, api, blockContent) {
+    getDsProcessor(ds, api) {
         const store = worker_threads_1.isMainThread
             ? this.storeService.getStore()
             : (0, node_core_1.hostStoreToStore)(global.host); // Provided in worker.ts
@@ -36,32 +35,29 @@ let SandboxService = class SandboxService {
         let processor = this.processorCache[entry];
         if (!processor) {
             processor = new node_core_1.IndexerSandbox({
-                // api: await this.apiService.getPatchedApi(),
                 store,
                 root: this.project.root,
-                // script: ds.mapping.entryScript,
                 entry,
+                chainId: this.project.network.chainId,
             }, this.nodeConfig);
             this.processorCache[entry] = processor;
         }
-        api.freezeApi(processor, blockContent);
+        processor.freeze(api, 'api');
+        if (this.nodeConfig.unsafe) {
+            processor.freeze(this.apiService.api.api, 'unsafeApi');
+        }
         return processor;
     }
     getDataSourceEntry(ds) {
-        if ((0, common_ethereum_1.isDatasourceV0_2_0)(ds)) {
-            return ds.mapping.file;
-        }
-        else {
-            return (0, project_1.getProjectEntry)(this.project.root);
-        }
+        return ds.mapping.file;
     }
 };
 SandboxService = __decorate([
     (0, common_1.Injectable)(),
-    __param(2, (0, common_1.Inject)('ISubqueryProject')),
-    __metadata("design:paramtypes", [node_core_1.StoreService,
-        node_core_1.NodeConfig,
-        SubqueryProject_1.SubqueryProject])
+    __param(3, (0, common_1.Inject)('ISubqueryProject')),
+    __metadata("design:paramtypes", [node_core_1.ApiService,
+        node_core_1.StoreService,
+        node_core_1.NodeConfig, Object])
 ], SandboxService);
 exports.SandboxService = SandboxService;
 //# sourceMappingURL=sandbox.service.js.map

package/dist/indexer/sandbox.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"sandbox.service.js","sourceRoot":"","sources":["../../src/indexer/sandbox.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,mDAA8C;AAC9C,2CAAoD;AACpD,4DAGgC;AAChC,gDAK0B;AAG1B,kEAA+E;AAC/E,8CAAmD;AAG5C,IAAM,cAAc,GAApB,MAAM,cAAc;IAGzB,YACmB,YAA0B,EAC1B,UAAsB,EACM,OAAwB;QAFpD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QAL/D,mBAAc,GAAmC,EAAE,CAAC;IAMzD,CAAC;IAEJ,qBAAqB,CACnB,EAAkB,EAClB,GAAe,EACf,YAAkC;QAElC,MAAM,KAAK,GAAU,6BAAY;YAC/B,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE;YAC9B,CAAC,CAAC,IAAA,4BAAgB,EAAE,MAAc,CAAC,IAAI,CAAC,CAAC,CAAC,wBAAwB;QAEpE,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,SAAS,EAAE;YACd,SAAS,GAAG,IAAI,0BAAc,CAC5B;gBACE,8CAA8C;gBAC9C,KAAK;gBACL,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;gBACvB,kCAAkC;gBAClC,KAAK;aACN,EACD,IAAI,CAAC,UAAU,CAChB,CAAC;YACF,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;SACxC;QACD,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACvC,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,kBAAkB,CAAC,EAA2B;QACpD,IAAI,IAAA,oCAAkB,EAAC,EAAE,CAAC,EAAE;YAC1B,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;SACxB;aAAM;YACL,OAAO,IAAA,yBAAe,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC3C;IACH,CAAC;CACF,CAAA;AA5CY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAOR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAFI,wBAAY;QACd,sBAAU;QACe,iCAAe;GAN5D,cAAc,CA4C1B;AA5CY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  isDatasourceV0_2_0,\n  SubqlEthereumDataSource,\n} from '@subql/common-ethereum';\nimport {\n  NodeConfig,\n  StoreService,\n  IndexerSandbox,\n  hostStoreToStore,\n} from '@subql/node-core';\nimport { Store } from '@subql/types';\nimport { ApiWrapper, EthereumBlockWrapper } from '@subql/types-ethereum';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { getProjectEntry } from '../utils/project';\n\n@Injectable()\nexport class SandboxService {\n  private processorCache: Record<string, IndexerSandbox> = {};\n\n  constructor(\n    private readonly storeService: StoreService,\n    private readonly nodeConfig: NodeConfig,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n  ) {}\n\n  getDsProcessorWrapper(\n    ds: SubqlProjectDs,\n    api: ApiWrapper,\n    blockContent: EthereumBlockWrapper,\n  ): IndexerSandbox {\n    const store: Store = isMainThread\n      ? this.storeService.getStore()\n      : hostStoreToStore((global as any).host); // Provided in worker.ts\n\n    const entry = this.getDataSourceEntry(ds);\n    let processor = this.processorCache[entry];\n    if (!processor) {\n      processor = new IndexerSandbox(\n        {\n          // api: await this.apiService.getPatchedApi(),\n          store,\n          root: this.project.root,\n          // script: ds.mapping.entryScript,\n          entry,\n        },\n        this.nodeConfig,\n      );\n      this.processorCache[entry] = processor;\n    }\n    api.freezeApi(processor, blockContent);\n    return processor;\n  }\n\n  private getDataSourceEntry(ds: SubqlEthereumDataSource): string {\n    if (isDatasourceV0_2_0(ds)) {\n      return ds.mapping.file;\n    } else {\n      return getProjectEntry(this.project.root);\n    }\n  }\n}\n"]}
+{"version":3,"file":"sandbox.service.js","sourceRoot":"","sources":["../../src/indexer/sandbox.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,mDAA8C;AAC9C,2CAAoD;AAEpD,gDAO0B;AAI1B,+GAA+G;AAExG,IAAM,cAAc,GAApB,MAAM,cAAc;IAGzB,YACmB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACM,OAAyB;QAHrD,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAkB;QANhE,mBAAc,GAAmC,EAAE,CAAC;IAOzD,CAAC;IAEJ,cAAc,CAAC,EAAkB,EAAE,GAAoB;QACrD,MAAM,KAAK,GAAU,6BAAY;YAC/B,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE;YAC9B,CAAC,CAAC,IAAA,4BAAgB,EAAE,MAAc,CAAC,IAAI,CAAC,CAAC,CAAC,wBAAwB;QAEpE,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,SAAS,EAAE;YACd,SAAS,GAAG,IAAI,0BAAc,CAC5B;gBACE,KAAK;gBACL,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;gBACvB,KAAK;gBACL,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO;aACtC,EACD,IAAI,CAAC,UAAU,CAChB,CAAC;YACF,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;SACxC;QAED,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;YAC1B,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;SACxD;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,kBAAkB,CAAC,EAAkB;QAC3C,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACzB,CAAC;CACF,CAAA;AAxCY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAQR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAHE,sBAAU;QACR,wBAAY;QACd,sBAAU;GAN9B,cAAc,CAwC1B;AAxCY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport { BaseDataSource } from '@subql/common';\nimport {\n  NodeConfig,\n  StoreService,\n  IndexerSandbox,\n  hostStoreToStore,\n  ISubqueryProject,\n  ApiService,\n} from '@subql/node-core';\nimport { Store } from '@subql/types-ethereum';\nimport SafeEthProvider from '../ethereum/safe-api';\n\n/* It would be nice to move this to node core but need to find a way to inject other things into the sandbox */\n@Injectable()\nexport class SandboxService {\n  private processorCache: Record<string, IndexerSandbox> = {};\n\n  constructor(\n    private readonly apiService: ApiService,\n    private readonly storeService: StoreService,\n    private readonly nodeConfig: NodeConfig,\n    @Inject('ISubqueryProject') private readonly project: ISubqueryProject,\n  ) {}\n\n  getDsProcessor(ds: BaseDataSource, api: SafeEthProvider): IndexerSandbox {\n    const store: Store = isMainThread\n      ? this.storeService.getStore()\n      : hostStoreToStore((global as any).host); // Provided in worker.ts\n\n    const entry = this.getDataSourceEntry(ds);\n    let processor = this.processorCache[entry];\n    if (!processor) {\n      processor = new IndexerSandbox(\n        {\n          store,\n          root: this.project.root,\n          entry,\n          chainId: this.project.network.chainId,\n        },\n        this.nodeConfig,\n      );\n      this.processorCache[entry] = processor;\n    }\n\n    processor.freeze(api, 'api');\n    if (this.nodeConfig.unsafe) {\n      processor.freeze(this.apiService.api.api, 'unsafeApi');\n    }\n    return processor;\n  }\n\n  private getDataSourceEntry(ds: BaseDataSource): string {\n    return ds.mapping.file;\n  }\n}\n"]}

package/dist/indexer/unfinalizedBlocks.service.d.ts

@@ -1,36 +1,13 @@
 import { Block } from '@ethersproject/abstract-provider';
-import { ApiService, NodeConfig, StoreCacheService } from '@subql/node-core';
-import { EthereumBlock } from '@subql/types-ethereum';
-export declare const METADATA_UNFINALIZED_BLOCKS_KEY = "unfinalizedBlocks";
-export declare const METADATA_LAST_FINALIZED_PROCESSED_KEY = "lastFinalizedVerifiedHeight";
-type UnfinalizedBlocks = [blockHeight: number, blockHash: string][];
-export interface IUnfinalizedBlocksService {
-    processUnfinalizedBlocks(block: EthereumBlock | undefined): Promise<number | null>;
-}
-export declare class UnfinalizedBlocksService implements IUnfinalizedBlocksService {
+import { ApiService, BaseUnfinalizedBlocksService, Header, NodeConfig, StoreCacheService } from '@subql/node-core';
+import { BlockWrapper, EthereumBlock } from '@subql/types-ethereum';
+import { EthereumApi } from '../ethereum';
+export declare function blockToHeader(block: EthereumBlock | Block): Header;
+export declare class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService<BlockWrapper> {
     private readonly apiService;
-    private readonly nodeConfig;
-    private readonly storeCache;
-    private unfinalizedBlocks;
-    private finalizedHeader;
-    private lastCheckedBlockHeight;
-    constructor(apiService: ApiService, nodeConfig: NodeConfig, storeCache: StoreCacheService);
-    init(reindex: (targetHeight: number) => Promise<void>): Promise<number | undefined>;
-    processUnfinalizedBlocks(block: EthereumBlock | undefined): Promise<number | null>;
-    private get api();
-    private get finalizedBlockNumber();
-    registerFinalizedBlock(header: Block): void;
-    private registerUnfinalizedBlock;
-    private deleteFinalizedBlock;
-    private removeFinalized;
-    private getClosestRecord;
-    private hasForked;
-    private getLastCorrectFinalizedBlock;
-    private saveUnfinalizedBlocks;
-    private saveLastFinalizedVerifiedHeight;
-    resetUnfinalizedBlocks(): void;
-    resetLastFinalizedVerifiedHeight(): void;
-    getMetadataUnfinalizedBlocks(): Promise<UnfinalizedBlocks>;
-    getLastFinalizedVerifiedHeight(): Promise<number | undefined>;
+    constructor(apiService: ApiService<any, EthereumApi>, nodeConfig: NodeConfig, storeCache: StoreCacheService);
+    protected blockToHeader(block: BlockWrapper): Header;
+    protected getFinalizedHead(): Promise<Header>;
+    protected getHeaderForHash(hash: string): Promise<Header>;
+    protected getHeaderForHeight(height: number): Promise<Header>;
 }
-export {};

package/dist/indexer/unfinalizedBlocks.service.js

@@ -11,180 +11,37 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.UnfinalizedBlocksService = exports.METADATA_LAST_FINALIZED_PROCESSED_KEY = exports.METADATA_UNFINALIZED_BLOCKS_KEY = void 0;
+exports.UnfinalizedBlocksService = exports.blockToHeader = void 0;
 const common_1 = require("@nestjs/common");
 const node_core_1 = require("@subql/node-core");
-const lodash_1 = require("lodash");
-const logger = (0, node_core_1.getLogger)('UnfinalizedBlocks');
-exports.METADATA_UNFINALIZED_BLOCKS_KEY = 'unfinalizedBlocks';
-exports.METADATA_LAST_FINALIZED_PROCESSED_KEY = 'lastFinalizedVerifiedHeight';
-const UNFINALIZED_THRESHOLD = 200;
-let UnfinalizedBlocksService = class UnfinalizedBlocksService {
+function blockToHeader(block) {
+    return {
+        blockHeight: block.number,
+        blockHash: block.hash,
+        parentHash: block.parentHash,
+    };
+}
+exports.blockToHeader = blockToHeader;
+let UnfinalizedBlocksService = class UnfinalizedBlocksService extends node_core_1.BaseUnfinalizedBlocksService {
     constructor(apiService, nodeConfig, storeCache) {
+        super(nodeConfig, storeCache);
         this.apiService = apiService;
-        this.nodeConfig = nodeConfig;
-        this.storeCache = storeCache;
     }
-    async init(reindex) {
-        logger.info(`Unfinalized blocks is ${this.nodeConfig.unfinalizedBlocks ? 'enabled' : 'disabled'}`);
-        // unfinalized blocks
-        this.unfinalizedBlocks = await this.getMetadataUnfinalizedBlocks();
-        this.lastCheckedBlockHeight = await this.getLastFinalizedVerifiedHeight();
-        this.finalizedHeader = await this.api.getBlockByHeightOrHash(await this.api.getFinalizedBlockHeight());
-        if (!this.nodeConfig.unfinalizedBlocks && this.unfinalizedBlocks.length) {
-            logger.info('Processing unfinalized blocks');
-            // Validate any previously unfinalized blocks
-            const rewindHeight = await this.processUnfinalizedBlocks(null);
-            if (rewindHeight !== null) {
-                logger.info(`Found un-finalized blocks from previous indexing but unverified, rolling back to last finalized block ${rewindHeight} `);
-                await reindex(rewindHeight);
-                logger.info(`Successful rewind to block ${rewindHeight}!`);
-                return rewindHeight;
-            }
-            else {
-                this.resetUnfinalizedBlocks();
-                this.resetLastFinalizedVerifiedHeight();
-            }
-        }
+    blockToHeader(block) {
+        return blockToHeader(block.block);
     }
-    async processUnfinalizedBlocks(block) {
-        if (block) {
-            this.registerUnfinalizedBlock(block.number, block.hash);
-        }
-        const forkedHeader = await this.hasForked();
-        if (!forkedHeader) {
-            // Remove blocks that are now confirmed finalized
-            this.deleteFinalizedBlock();
-        }
-        else {
-            // Get the last unfinalized block that is now finalized
-            return this.getLastCorrectFinalizedBlock(forkedHeader);
-        }
-        return null;
+    async getFinalizedHead() {
+        const finalizedHeight = await this.apiService.api.getFinalizedBlockHeight();
+        const finalizedBlock = await this.apiService.api.getBlockByHeightOrHash(finalizedHeight);
+        return blockToHeader(finalizedBlock);
     }
-    get api() {
-        return this.apiService.api;
+    async getHeaderForHash(hash) {
+        const block = await this.apiService.api.getBlockByHeightOrHash(hash);
+        return blockToHeader(block);
     }
-    get finalizedBlockNumber() {
-        return this.finalizedHeader.number;
-    }
-    registerFinalizedBlock(header) {
-        if (this.finalizedHeader && this.finalizedBlockNumber >= header.number) {
-            return;
-        }
-        this.finalizedHeader = header;
-    }
-    registerUnfinalizedBlock(blockNumber, hash) {
-        if (blockNumber <= this.finalizedBlockNumber)
-            return;
-        // Ensure order
-        if (this.unfinalizedBlocks.length &&
-            (0, lodash_1.last)(this.unfinalizedBlocks)[0] + 1 !== blockNumber) {
-            logger.error('Unfinalized block is not sequential');
-            process.exit(1);
-        }
-        this.unfinalizedBlocks.push([blockNumber, hash]);
-        this.saveUnfinalizedBlocks(this.unfinalizedBlocks);
-    }
-    deleteFinalizedBlock() {
-        if (this.lastCheckedBlockHeight !== undefined &&
-            this.lastCheckedBlockHeight < this.finalizedBlockNumber) {
-            this.removeFinalized(this.finalizedBlockNumber);
-            this.saveLastFinalizedVerifiedHeight(this.finalizedBlockNumber);
-            this.saveUnfinalizedBlocks(this.unfinalizedBlocks);
-        }
-        this.lastCheckedBlockHeight = this.finalizedBlockNumber;
-    }
-    // remove any records less and equal than input finalized blockHeight
-    removeFinalized(blockHeight) {
-        this.unfinalizedBlocks = this.unfinalizedBlocks.filter(([height]) => height > blockHeight);
-    }
-    // find closest record from block heights
-    getClosestRecord(blockHeight) {
-        // Have the block in the best block, can be verified
-        const record = [...this.unfinalizedBlocks] // Copy so we can reverse
-            .reverse() // Reverse the list to find the largest block
-            .find(([bestBlockHeight]) => Number(bestBlockHeight) <= blockHeight);
-        if (record) {
-            const [bestBlockHeight, hash] = record;
-            return { blockHeight: Number(bestBlockHeight), hash };
-        }
-        return undefined;
-    }
-    // check unfinalized blocks for a fork, returns the header where a fork happened
-    async hasForked() {
-        const lastVerifiableBlock = this.getClosestRecord(this.finalizedBlockNumber);
-        // No unfinalized blocks
-        if (!lastVerifiableBlock) {
-            return;
-        }
-        // Unfinalized blocks beyond finalized block
-        if (lastVerifiableBlock.blockHeight === this.finalizedBlockNumber) {
-            if (lastVerifiableBlock.hash !== this.finalizedHeader.hash) {
-                logger.warn(`Block fork found, enqueued un-finalized block at ${lastVerifiableBlock.blockHeight} with hash ${lastVerifiableBlock.hash}, actual hash is ${this.finalizedHeader.hash}`);
-                return this.finalizedHeader;
-            }
-        }
-        else {
-            // Unfinalized blocks below finalized block
-            let header = this.finalizedHeader;
-            /*
-             * Iterate back through parent hashes until we get the header with the matching height
-             * We use headers here rather than getBlockHash because of potential caching issues on the rpc
-             * If we're off by a large number of blocks we can optimise by getting the block hash directly
-             */
-            if (header.number - lastVerifiableBlock.blockHeight >
-                UNFINALIZED_THRESHOLD) {
-                header = await this.api.getBlockByHeightOrHash(lastVerifiableBlock.blockHeight);
-            }
-            else {
-                while (lastVerifiableBlock.blockHeight !== header.number) {
-                    header = await this.api.getBlockByHeightOrHash(header.parentHash);
-                }
-            }
-            if (header.hash !== lastVerifiableBlock.hash) {
-                logger.warn(`Block fork found, enqueued un-finalized block at ${lastVerifiableBlock.blockHeight} with hash ${lastVerifiableBlock.hash}, actual hash is ${header.hash}`);
-                return header;
-            }
-        }
-        return;
-    }
-    async getLastCorrectFinalizedBlock(forkedHeader) {
-        const bestVerifiableBlocks = this.unfinalizedBlocks.filter(([bestBlockHeight]) => Number(bestBlockHeight) <= this.finalizedBlockNumber);
-        let checkingHeader = forkedHeader;
-        // Work backwards through the blocks until we find a matching hash
-        for (const [block, hash] of bestVerifiableBlocks.reverse()) {
-            if (hash === checkingHeader.hash || hash === checkingHeader.parentHash) {
-                return Number(block);
-            }
-            // Get the new parent
-            checkingHeader = await this.api.getBlockByHeightOrHash(checkingHeader.parentHash);
-        }
-        return this.lastCheckedBlockHeight;
-    }
-    saveUnfinalizedBlocks(unfinalizedBlocks) {
-        return this.storeCache.metadata.set(exports.METADATA_UNFINALIZED_BLOCKS_KEY, JSON.stringify(unfinalizedBlocks));
-    }
-    saveLastFinalizedVerifiedHeight(height) {
-        return this.storeCache.metadata.set(exports.METADATA_LAST_FINALIZED_PROCESSED_KEY, height);
-    }
-    resetUnfinalizedBlocks() {
-        this.storeCache.metadata.set(exports.METADATA_UNFINALIZED_BLOCKS_KEY, '[]');
-        this.unfinalizedBlocks = [];
-    }
-    resetLastFinalizedVerifiedHeight() {
-        return this.storeCache.metadata.set(exports.METADATA_LAST_FINALIZED_PROCESSED_KEY, null);
-    }
-    //string should be jsonb object
-    async getMetadataUnfinalizedBlocks() {
-        const val = await this.storeCache.metadata.find(exports.METADATA_UNFINALIZED_BLOCKS_KEY);
-        if (val) {
-            return JSON.parse(val);
-        }
-        return [];
-    }
-    async getLastFinalizedVerifiedHeight() {
-        return this.storeCache.metadata.find(exports.METADATA_LAST_FINALIZED_PROCESSED_KEY);
+    async getHeaderForHeight(height) {
+        const block = await this.apiService.api.getBlockByHeightOrHash(height);
+        return blockToHeader(block);
     }
 };
 UnfinalizedBlocksService = __decorate([

package/dist/indexer/unfinalizedBlocks.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"unfinalizedBlocks.service.js","sourceRoot":"","sources":["../../src/indexer/unfinalizedBlocks.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAGtC,2CAA4C;AAC5C,gDAK0B;AAE1B,mCAA8B;AAG9B,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,CAAC,CAAC;AAEjC,QAAA,+BAA+B,GAAG,mBAAmB,CAAC;AACtD,QAAA,qCAAqC,GAChD,6BAA6B,CAAC;AAEhC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAU3B,IAAM,wBAAwB,GAA9B,MAAM,wBAAwB;IAKnC,YACmB,UAAsB,EACtB,UAAsB,EACtB,UAA6B;QAF7B,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAmB;IAC7C,CAAC;IAEJ,KAAK,CAAC,IAAI,CACR,OAAgD;QAEhD,MAAM,CAAC,IAAI,CACT,yBACE,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAClD,EAAE,CACH,CAAC;QACF,qBAAqB;QACrB,IAAI,CAAC,iBAAiB,GAAG,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACnE,IAAI,CAAC,sBAAsB,GAAG,MAAM,IAAI,CAAC,8BAA8B,EAAE,CAAC;QAC1E,IAAI,CAAC,eAAe,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC1D,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CACzC,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;YACvE,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YAC7C,6CAA6C;YAE7C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE/D,IAAI,YAAY,KAAK,IAAI,EAAE;gBACzB,MAAM,CAAC,IAAI,CACT,yGAAyG,YAAY,GAAG,CACzH,CAAC;gBACF,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,8BAA8B,YAAY,GAAG,CAAC,CAAC;gBAC3D,OAAO,YAAY,CAAC;aACrB;iBAAM;gBACL,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC9B,IAAI,CAAC,gCAAgC,EAAE,CAAC;aACzC;SACF;IACH,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,KAAgC;QAEhC,IAAI,KAAK,EAAE;YACT,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;SACzD;QAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5C,IAAI,CAAC,YAAY,EAAE;YACjB,iDAAiD;YACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;SAC7B;aAAM;YACL,uDAAuD;YACvD,OAAO,IAAI,CAAC,4BAA4B,CAAC,YAAY,CAAC,CAAC;SACxD;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAY,GAAG;QACb,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7B,CAAC;IAED,IAAY,oBAAoB;QAC9B,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,sBAAsB,CAAC,MAAa;QAClC,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE;YACtE,OAAO;SACR;QACD,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC;IAChC,CAAC;IAEO,wBAAwB,CAAC,WAAmB,EAAE,IAAY;QAChE,IAAI,WAAW,IAAI,IAAI,CAAC,oBAAoB;YAAE,OAAO;QAErD,eAAe;QACf,IACE,IAAI,CAAC,iBAAiB,CAAC,MAAM;YAC7B,IAAA,aAAI,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,WAAW,EACnD;YACA,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACrD,CAAC;IAEO,oBAAoB;QAC1B,IACE,IAAI,CAAC,sBAAsB,KAAK,SAAS;YACzC,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,oBAAoB,EACvD;YACA,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAChD,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAChE,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,oBAAoB,CAAC;IAC1D,CAAC;IAED,qEAAqE;IAC7D,eAAe,CAAC,WAAmB;QACzC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CACpD,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,GAAG,WAAW,CACnC,CAAC;IACJ,CAAC;IAED,yCAAyC;IACjC,gBAAgB,CACtB,WAAmB;QAEnB,oDAAoD;QACpD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,yBAAyB;aACjE,OAAO,EAAE,CAAC,6CAA6C;aACvD,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,WAAW,CAAC,CAAC;QACvE,IAAI,MAAM,EAAE;YACV,MAAM,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC;YACvC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,CAAC;SACvD;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,gFAAgF;IACxE,KAAK,CAAC,SAAS;QACrB,MAAM,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAC/C,IAAI,CAAC,oBAAoB,CAC1B,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,mBAAmB,EAAE;YACxB,OAAO;SACR;QAED,4CAA4C;QAC5C,IAAI,mBAAmB,CAAC,WAAW,KAAK,IAAI,CAAC,oBAAoB,EAAE;YACjE,IAAI,mBAAmB,CAAC,IAAI,KAAK,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE;gBAC1D,MAAM,CAAC,IAAI,CACT,oDAAoD,mBAAmB,CAAC,WAAW,cAAc,mBAAmB,CAAC,IAAI,oBAAoB,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CACzK,CAAC;gBACF,OAAO,IAAI,CAAC,eAAe,CAAC;aAC7B;SACF;aAAM;YACL,2CAA2C;YAC3C,IAAI,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC;YAClC;;;;eAIG;YACH,IACE,MAAM,CAAC,MAAM,GAAG,mBAAmB,CAAC,WAAW;gBAC/C,qBAAqB,EACrB;gBACA,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC5C,mBAAmB,CAAC,WAAW,CAChC,CAAC;aACH;iBAAM;gBACL,OAAO,mBAAmB,CAAC,WAAW,KAAK,MAAM,CAAC,MAAM,EAAE;oBACxD,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;iBACnE;aACF;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,mBAAmB,CAAC,IAAI,EAAE;gBAC5C,MAAM,CAAC,IAAI,CACT,oDAAoD,mBAAmB,CAAC,WAAW,cAAc,mBAAmB,CAAC,IAAI,oBAAoB,MAAM,CAAC,IAAI,EAAE,CAC3J,CAAC;gBACF,OAAO,MAAM,CAAC;aACf;SACF;QAED,OAAO;IACT,CAAC;IAEO,KAAK,CAAC,4BAA4B,CACxC,YAAmB;QAEnB,MAAM,oBAAoB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CACxD,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CACpB,MAAM,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,oBAAoB,CACvD,CAAC;QAEF,IAAI,cAAc,GAAG,YAAY,CAAC;QAElC,kEAAkE;QAClE,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,oBAAoB,CAAC,OAAO,EAAE,EAAE;YAC1D,IAAI,IAAI,KAAK,cAAc,CAAC,IAAI,IAAI,IAAI,KAAK,cAAc,CAAC,UAAU,EAAE;gBACtE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;aACtB;YAED,qBAAqB;YACrB,cAAc,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CACpD,cAAc,CAAC,UAAU,CAC1B,CAAC;SACH;QAED,OAAO,IAAI,CAAC,sBAAsB,CAAC;IACrC,CAAC;IAEO,qBAAqB,CAAC,iBAAoC;QAChE,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,uCAA+B,EAC/B,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAClC,CAAC;IACJ,CAAC;IAEO,+BAA+B,CAAC,MAAc;QACpD,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,6CAAqC,EACrC,MAAM,CACP,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,uCAA+B,EAAE,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED,gCAAgC;QAC9B,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,6CAAqC,EACrC,IAAI,CACL,CAAC;IACJ,CAAC;IAED,+BAA+B;IAC/B,KAAK,CAAC,4BAA4B;QAChC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAC7C,uCAA+B,CAChC,CAAC;QACF,IAAI,GAAG,EAAE;YACP,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAsB,CAAC;SAC7C;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,8BAA8B;QAClC,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,6CAAqC,CAAC,CAAC;IAC9E,CAAC;CACF,CAAA;AAtPY,wBAAwB;IADpC,IAAA,mBAAU,GAAE;qCAOoB,sBAAU;QACV,sBAAU;QACV,6BAAiB;GARrC,wBAAwB,CAsPpC;AAtPY,4DAAwB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Block } from '@ethersproject/abstract-provider';\nimport { Injectable } from '@nestjs/common';\nimport {\n  ApiService,\n  getLogger,\n  NodeConfig,\n  StoreCacheService,\n} from '@subql/node-core';\nimport { EthereumBlock } from '@subql/types-ethereum';\nimport { last } from 'lodash';\nimport { EthereumApi } from '../ethereum';\n\nconst logger = getLogger('UnfinalizedBlocks');\n\nexport const METADATA_UNFINALIZED_BLOCKS_KEY = 'unfinalizedBlocks';\nexport const METADATA_LAST_FINALIZED_PROCESSED_KEY =\n  'lastFinalizedVerifiedHeight';\n\nconst UNFINALIZED_THRESHOLD = 200;\n\ntype UnfinalizedBlocks = [blockHeight: number, blockHash: string][];\nexport interface IUnfinalizedBlocksService {\n  processUnfinalizedBlocks(\n    block: EthereumBlock | undefined,\n  ): Promise<number | null>;\n}\n\n@Injectable()\nexport class UnfinalizedBlocksService implements IUnfinalizedBlocksService {\n  private unfinalizedBlocks: UnfinalizedBlocks;\n  private finalizedHeader: Block;\n  private lastCheckedBlockHeight: number;\n\n  constructor(\n    private readonly apiService: ApiService,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeCache: StoreCacheService,\n  ) {}\n\n  async init(\n    reindex: (targetHeight: number) => Promise<void>,\n  ): Promise<number | undefined> {\n    logger.info(\n      `Unfinalized blocks is ${\n        this.nodeConfig.unfinalizedBlocks ? 'enabled' : 'disabled'\n      }`,\n    );\n    // unfinalized blocks\n    this.unfinalizedBlocks = await this.getMetadataUnfinalizedBlocks();\n    this.lastCheckedBlockHeight = await this.getLastFinalizedVerifiedHeight();\n    this.finalizedHeader = await this.api.getBlockByHeightOrHash(\n      await this.api.getFinalizedBlockHeight(),\n    );\n\n    if (!this.nodeConfig.unfinalizedBlocks && this.unfinalizedBlocks.length) {\n      logger.info('Processing unfinalized blocks');\n      // Validate any previously unfinalized blocks\n\n      const rewindHeight = await this.processUnfinalizedBlocks(null);\n\n      if (rewindHeight !== null) {\n        logger.info(\n          `Found un-finalized blocks from previous indexing but unverified, rolling back to last finalized block ${rewindHeight} `,\n        );\n        await reindex(rewindHeight);\n        logger.info(`Successful rewind to block ${rewindHeight}!`);\n        return rewindHeight;\n      } else {\n        this.resetUnfinalizedBlocks();\n        this.resetLastFinalizedVerifiedHeight();\n      }\n    }\n  }\n\n  async processUnfinalizedBlocks(\n    block: EthereumBlock | undefined,\n  ): Promise<number | null> {\n    if (block) {\n      this.registerUnfinalizedBlock(block.number, block.hash);\n    }\n\n    const forkedHeader = await this.hasForked();\n    if (!forkedHeader) {\n      // Remove blocks that are now confirmed finalized\n      this.deleteFinalizedBlock();\n    } else {\n      // Get the last unfinalized block that is now finalized\n      return this.getLastCorrectFinalizedBlock(forkedHeader);\n    }\n\n    return null;\n  }\n\n  private get api(): EthereumApi {\n    return this.apiService.api;\n  }\n\n  private get finalizedBlockNumber(): number {\n    return this.finalizedHeader.number;\n  }\n\n  registerFinalizedBlock(header: Block): void {\n    if (this.finalizedHeader && this.finalizedBlockNumber >= header.number) {\n      return;\n    }\n    this.finalizedHeader = header;\n  }\n\n  private registerUnfinalizedBlock(blockNumber: number, hash: string): void {\n    if (blockNumber <= this.finalizedBlockNumber) return;\n\n    // Ensure order\n    if (\n      this.unfinalizedBlocks.length &&\n      last(this.unfinalizedBlocks)[0] + 1 !== blockNumber\n    ) {\n      logger.error('Unfinalized block is not sequential');\n      process.exit(1);\n    }\n\n    this.unfinalizedBlocks.push([blockNumber, hash]);\n    this.saveUnfinalizedBlocks(this.unfinalizedBlocks);\n  }\n\n  private deleteFinalizedBlock(): void {\n    if (\n      this.lastCheckedBlockHeight !== undefined &&\n      this.lastCheckedBlockHeight < this.finalizedBlockNumber\n    ) {\n      this.removeFinalized(this.finalizedBlockNumber);\n      this.saveLastFinalizedVerifiedHeight(this.finalizedBlockNumber);\n      this.saveUnfinalizedBlocks(this.unfinalizedBlocks);\n    }\n    this.lastCheckedBlockHeight = this.finalizedBlockNumber;\n  }\n\n  // remove any records less and equal than input finalized blockHeight\n  private removeFinalized(blockHeight: number): void {\n    this.unfinalizedBlocks = this.unfinalizedBlocks.filter(\n      ([height]) => height > blockHeight,\n    );\n  }\n\n  // find closest record from block heights\n  private getClosestRecord(\n    blockHeight: number,\n  ): { blockHeight: number; hash: string } | undefined {\n    // Have the block in the best block, can be verified\n    const record = [...this.unfinalizedBlocks] // Copy so we can reverse\n      .reverse() // Reverse the list to find the largest block\n      .find(([bestBlockHeight]) => Number(bestBlockHeight) <= blockHeight);\n    if (record) {\n      const [bestBlockHeight, hash] = record;\n      return { blockHeight: Number(bestBlockHeight), hash };\n    }\n    return undefined;\n  }\n\n  // check unfinalized blocks for a fork, returns the header where a fork happened\n  private async hasForked(): Promise<Block | undefined> {\n    const lastVerifiableBlock = this.getClosestRecord(\n      this.finalizedBlockNumber,\n    );\n\n    // No unfinalized blocks\n    if (!lastVerifiableBlock) {\n      return;\n    }\n\n    // Unfinalized blocks beyond finalized block\n    if (lastVerifiableBlock.blockHeight === this.finalizedBlockNumber) {\n      if (lastVerifiableBlock.hash !== this.finalizedHeader.hash) {\n        logger.warn(\n          `Block fork found, enqueued un-finalized block at ${lastVerifiableBlock.blockHeight} with hash ${lastVerifiableBlock.hash}, actual hash is ${this.finalizedHeader.hash}`,\n        );\n        return this.finalizedHeader;\n      }\n    } else {\n      // Unfinalized blocks below finalized block\n      let header = this.finalizedHeader;\n      /*\n       * Iterate back through parent hashes until we get the header with the matching height\n       * We use headers here rather than getBlockHash because of potential caching issues on the rpc\n       * If we're off by a large number of blocks we can optimise by getting the block hash directly\n       */\n      if (\n        header.number - lastVerifiableBlock.blockHeight >\n        UNFINALIZED_THRESHOLD\n      ) {\n        header = await this.api.getBlockByHeightOrHash(\n          lastVerifiableBlock.blockHeight,\n        );\n      } else {\n        while (lastVerifiableBlock.blockHeight !== header.number) {\n          header = await this.api.getBlockByHeightOrHash(header.parentHash);\n        }\n      }\n\n      if (header.hash !== lastVerifiableBlock.hash) {\n        logger.warn(\n          `Block fork found, enqueued un-finalized block at ${lastVerifiableBlock.blockHeight} with hash ${lastVerifiableBlock.hash}, actual hash is ${header.hash}`,\n        );\n        return header;\n      }\n    }\n\n    return;\n  }\n\n  private async getLastCorrectFinalizedBlock(\n    forkedHeader: Block,\n  ): Promise<number | undefined> {\n    const bestVerifiableBlocks = this.unfinalizedBlocks.filter(\n      ([bestBlockHeight]) =>\n        Number(bestBlockHeight) <= this.finalizedBlockNumber,\n    );\n\n    let checkingHeader = forkedHeader;\n\n    // Work backwards through the blocks until we find a matching hash\n    for (const [block, hash] of bestVerifiableBlocks.reverse()) {\n      if (hash === checkingHeader.hash || hash === checkingHeader.parentHash) {\n        return Number(block);\n      }\n\n      // Get the new parent\n      checkingHeader = await this.api.getBlockByHeightOrHash(\n        checkingHeader.parentHash,\n      );\n    }\n\n    return this.lastCheckedBlockHeight;\n  }\n\n  private saveUnfinalizedBlocks(unfinalizedBlocks: UnfinalizedBlocks): void {\n    return this.storeCache.metadata.set(\n      METADATA_UNFINALIZED_BLOCKS_KEY,\n      JSON.stringify(unfinalizedBlocks),\n    );\n  }\n\n  private saveLastFinalizedVerifiedHeight(height: number): void {\n    return this.storeCache.metadata.set(\n      METADATA_LAST_FINALIZED_PROCESSED_KEY,\n      height,\n    );\n  }\n\n  resetUnfinalizedBlocks(): void {\n    this.storeCache.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, '[]');\n    this.unfinalizedBlocks = [];\n  }\n\n  resetLastFinalizedVerifiedHeight(): void {\n    return this.storeCache.metadata.set(\n      METADATA_LAST_FINALIZED_PROCESSED_KEY,\n      null,\n    );\n  }\n\n  //string should be jsonb object\n  async getMetadataUnfinalizedBlocks(): Promise<UnfinalizedBlocks> {\n    const val = await this.storeCache.metadata.find(\n      METADATA_UNFINALIZED_BLOCKS_KEY,\n    );\n    if (val) {\n      return JSON.parse(val) as UnfinalizedBlocks;\n    }\n    return [];\n  }\n\n  async getLastFinalizedVerifiedHeight(): Promise<number | undefined> {\n    return this.storeCache.metadata.find(METADATA_LAST_FINALIZED_PROCESSED_KEY);\n  }\n}\n"]}
+{"version":3,"file":"unfinalizedBlocks.service.js","sourceRoot":"","sources":["../../src/indexer/unfinalizedBlocks.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAGtC,2CAA4C;AAC5C,gDAM0B;AAI1B,SAAgB,aAAa,CAAC,KAA4B;IACxD,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,MAAM;QACzB,SAAS,EAAE,KAAK,CAAC,IAAI;QACrB,UAAU,EAAE,KAAK,CAAC,UAAU;KAC7B,CAAC;AACJ,CAAC;AAND,sCAMC;AAGM,IAAM,wBAAwB,GAA9B,MAAM,wBAAyB,SAAQ,wCAA0C;IACtF,YACmB,UAAwC,EACzD,UAAsB,EACtB,UAA6B;QAE7B,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAJb,eAAU,GAAV,UAAU,CAA8B;IAK3D,CAAC;IAES,aAAa,CAAC,KAAmB;QACzC,OAAO,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,gBAAgB;QAC9B,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC;QAC5E,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,sBAAsB,CACrE,eAAe,CAChB,CAAC;QACF,OAAO,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,IAAY;QAC3C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;QACrE,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAC,MAAc;QAC/C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;QACvE,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;CACF,CAAA;AA9BY,wBAAwB;IADpC,IAAA,mBAAU,GAAE;qCAGoB,sBAAU;QAC3B,sBAAU;QACV,6BAAiB;GAJpB,wBAAwB,CA8BpC;AA9BY,4DAAwB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Block } from '@ethersproject/abstract-provider';\nimport { Injectable } from '@nestjs/common';\nimport {\n  ApiService,\n  BaseUnfinalizedBlocksService,\n  Header,\n  NodeConfig,\n  StoreCacheService,\n} from '@subql/node-core';\nimport { BlockWrapper, EthereumBlock } from '@subql/types-ethereum';\nimport { EthereumApi } from '../ethereum';\n\nexport function blockToHeader(block: EthereumBlock | Block): Header {\n  return {\n    blockHeight: block.number,\n    blockHash: block.hash,\n    parentHash: block.parentHash,\n  };\n}\n\n@Injectable()\nexport class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService<BlockWrapper> {\n  constructor(\n    private readonly apiService: ApiService<any, EthereumApi>,\n    nodeConfig: NodeConfig,\n    storeCache: StoreCacheService,\n  ) {\n    super(nodeConfig, storeCache);\n  }\n\n  protected blockToHeader(block: BlockWrapper): Header {\n    return blockToHeader(block.block);\n  }\n\n  protected async getFinalizedHead(): Promise<Header> {\n    const finalizedHeight = await this.apiService.api.getFinalizedBlockHeight();\n    const finalizedBlock = await this.apiService.api.getBlockByHeightOrHash(\n      finalizedHeight,\n    );\n    return blockToHeader(finalizedBlock);\n  }\n\n  protected async getHeaderForHash(hash: string): Promise<Header> {\n    const block = await this.apiService.api.getBlockByHeightOrHash(hash);\n    return blockToHeader(block);\n  }\n\n  protected async getHeaderForHeight(height: number): Promise<Header> {\n    const block = await this.apiService.api.getBlockByHeightOrHash(height);\n    return blockToHeader(block);\n  }\n}\n"]}

package/dist/indexer/worker/worker.d.ts

@@ -1,4 +1,5 @@
-import { FetchBlockResponse, ProcessBlockResponse, WorkerStatusResponse } from './worker.service';
+import { ProcessBlockResponse } from '@subql/node-core';
+import { FetchBlockResponse, WorkerStatusResponse } from './worker.service';
 declare function initWorker(): Promise<void>;
 declare function fetchBlock(height: number): Promise<FetchBlockResponse>;
 declare function processBlock(height: number): Promise<ProcessBlockResponse>;

package/dist/indexer/worker/worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;AAEtC,iEAAiE;AACjE,kDAAkD;AAClD,wCAAwC;AACxC,oDAAqD;AACrD,wCAAwC;AACxC,uCAA2C;AAE3C,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oDAA4B;AAC5B,6DAA+C;AAC/C,2BAAuC;AAEvC,uCAA2C;AAC3C,gDAS0B;AAE1B,wDAAoD;AACpD,mDAA+C;AAC/C,qDAK0B;AAC1B,yFAG4C;AAC5C,IAAI,GAAqB,CAAC;AAC1B,IAAI,aAA4B,CAAC;AAEjC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,WAAW,8BAAQ,EAAE,CAAC,CAAC;AAEhD,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,IAAI,GAAG,EAAE;YACP,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC1C,OAAO;SACR;QAED,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,4BAAY,EAAE;YAC3C,MAAM,EAAE,IAAI,sBAAU,EAAE,EAAE,0EAA0E;SACrG,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,uGAAuG;QACvG,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,8BAAa,CAAC,CAAC;KACxC;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC1C,MAAM,CAAC,CAAC;KACT;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,MAAc;IACtC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IACzC,OAAO,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAc;IACxC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,OAAO,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,gBAAgB;IAC7B,OAAO,aAAa,CAAC,gBAAgB,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,iBAAiB;IAC9B,OAAO,aAAa,CAAC,iBAAiB,CAAC;AACzC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,SAAS;IACtB,OAAO;QACL,QAAQ,EAAR,8BAAQ;QACR,aAAa,EAAE,aAAa,CAAC,gBAAgB;QAC7C,aAAa,EAAE,aAAa,CAAC,iBAAiB;QAC9C,UAAU,EAAE,aAAa,CAAC,UAAU;KACrC,CAAC;AACJ,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,aAAa;IAC1B,MAAM,SAAS,GAAG,IAAA,sBAAiB,GAAE,CAAC,eAAe,CAAC;IACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;IAEhD,OAAO,SAAS,GAAG,QAAQ,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,eAAuB;IAC3D,MAAM,IAAA,4BAAgB,EAAC,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED,wDAAwD;AACvD,MAAc,CAAC,IAAI,GAAG,sBAAU,CAAC,MAAM,CAItC,CAAC,GAAG,yBAAa,EAAE,GAAG,6BAAiB,EAAE,GAAG,4DAAyB,CAAC,EACtE;IACE,UAAU;IACV,UAAU;IACV,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,SAAS;IACT,aAAa;IACb,sBAAsB;CACvB,EACD,MAAM,CACP,CAAC;AA0BF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;IACpC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtC,MAAM,CAAC,CAAC;AACV,CAAC,CAAC,CAAC","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\n// initlogger and yargs must be imported before all other imports\n// making sure logger is defined before its called\n// eslint-disable-next-line import/order\nimport { initLogger } from '@subql/node-core/logger';\n// eslint-disable-next-line import/order\nimport { yargsOptions } from '../../yargs';\n\nconst { argv } = yargsOptions;\n\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\nimport assert from 'assert';\nimport { threadId } from 'node:worker_threads';\nimport { getHeapStatistics } from 'v8';\nimport { INestApplication } from '@nestjs/common';\nimport { NestFactory } from '@nestjs/core';\nimport {\n  waitForBatchSize,\n  WorkerHost,\n  getLogger,\n  NestLogger,\n  hostStoreKeys,\n  HostStore,\n  hostDynamicDsKeys,\n  HostDynamicDS,\n} from '@subql/node-core';\nimport { SubqlProjectDs } from '../../configure/SubqueryProject';\nimport { IndexerManager } from '../indexer.manager';\nimport { WorkerModule } from './worker.module';\nimport {\n  FetchBlockResponse,\n  ProcessBlockResponse,\n  WorkerService,\n  WorkerStatusResponse,\n} from './worker.service';\nimport {\n  HostUnfinalizedBlocks,\n  hostUnfinalizedBlocksKeys,\n} from './worker.unfinalizedBlocks.service';\nlet app: INestApplication;\nlet workerService: WorkerService;\n\nconst logger = getLogger(`worker #${threadId}`);\n\nasync function initWorker(): Promise<void> {\n  try {\n    if (app) {\n      logger.warn('Worker already initialised');\n      return;\n    }\n\n    app = await NestFactory.create(WorkerModule, {\n      logger: new NestLogger(), // TIP: If the worker is crashing comment out this line for better logging\n    });\n\n    await app.init();\n\n    const indexerManager = app.get(IndexerManager);\n    // Initialise async services, we do this here rather than in factories so we can capture one off events\n    await indexerManager.start();\n\n    workerService = app.get(WorkerService);\n  } catch (e) {\n    console.log('Failed to start worker', e);\n    logger.error(e, 'Failed to start worker');\n    throw e;\n  }\n}\n\nasync function fetchBlock(height: number): Promise<FetchBlockResponse> {\n  assert(workerService, 'Not initialised');\n  return workerService.fetchBlock(height);\n}\n\nasync function processBlock(height: number): Promise<ProcessBlockResponse> {\n  assert(workerService, 'Not initialised');\n\n  return workerService.processBlock(height);\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchedBlocks(): Promise<number> {\n  return workerService.numFetchedBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchingBlocks(): Promise<number> {\n  return workerService.numFetchingBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getStatus(): Promise<WorkerStatusResponse> {\n  return {\n    threadId,\n    fetchedBlocks: workerService.numFetchedBlocks,\n    toFetchBlocks: workerService.numFetchingBlocks,\n    isIndexing: workerService.isIndexing,\n  };\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getMemoryLeft(): Promise<number> {\n  const totalHeap = getHeapStatistics().heap_size_limit;\n  const heapUsed = process.memoryUsage().heapUsed;\n\n  return totalHeap - heapUsed;\n}\n\nasync function waitForWorkerBatchSize(heapSizeInBytes: number): Promise<void> {\n  await waitForBatchSize(heapSizeInBytes);\n}\n\n// Register these functions to be exposed to worker host\n(global as any).host = WorkerHost.create<\n  HostStore & HostDynamicDS<SubqlProjectDs> & HostUnfinalizedBlocks,\n  IInitIndexerWorker\n>(\n  [...hostStoreKeys, ...hostDynamicDsKeys, ...hostUnfinalizedBlocksKeys],\n  {\n    initWorker,\n    fetchBlock,\n    processBlock,\n    numFetchedBlocks,\n    numFetchingBlocks,\n    getStatus,\n    getMemoryLeft,\n    waitForWorkerBatchSize,\n  },\n  logger,\n);\n\n// Export types to be used on the parent\ntype InitWorker = typeof initWorker;\ntype FetchBlock = typeof fetchBlock;\ntype ProcessBlock = typeof processBlock;\ntype NumFetchedBlocks = typeof numFetchedBlocks;\ntype NumFetchingBlocks = typeof numFetchingBlocks;\ntype GetWorkerStatus = typeof getStatus;\ntype GetMemoryLeft = typeof getMemoryLeft;\ntype WaitForWorkerBatchSize = typeof waitForWorkerBatchSize;\n\nexport type IIndexerWorker = {\n  processBlock: ProcessBlock;\n  fetchBlock: FetchBlock;\n  numFetchedBlocks: NumFetchedBlocks;\n  numFetchingBlocks: NumFetchingBlocks;\n  getStatus: GetWorkerStatus;\n  getMemoryLeft: GetMemoryLeft;\n  waitForWorkerBatchSize: WaitForWorkerBatchSize;\n};\n\nexport type IInitIndexerWorker = IIndexerWorker & {\n  initWorker: InitWorker;\n};\n\nprocess.on('uncaughtException', (e) => {\n  logger.error(e, 'Uncaught Exception');\n  throw e;\n});\n"]}
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;AAEtC,iEAAiE;AACjE,kDAAkD;AAClD,wCAAwC;AACxC,oDAAqD;AACrD,wCAAwC;AACxC,uCAA2C;AAE3C,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oDAA4B;AAC5B,6DAA+C;AAC/C,2BAAuC;AAEvC,uCAA2C;AAC3C,gDAU0B;AAE1B,wDAAoD;AACpD,mDAA+C;AAC/C,qDAI0B;AAC1B,yFAG4C;AAC5C,IAAI,GAAqB,CAAC;AAC1B,IAAI,aAA4B,CAAC;AAEjC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,WAAW,8BAAQ,EAAE,CAAC,CAAC;AAEhD,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,IAAI,GAAG,EAAE;YACP,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC1C,OAAO;SACR;QAED,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,4BAAY,EAAE;YAC3C,MAAM,EAAE,IAAI,sBAAU,EAAE,EAAE,0EAA0E;SACrG,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,uGAAuG;QACvG,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,8BAAa,CAAC,CAAC;KACxC;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC1C,MAAM,CAAC,CAAC;KACT;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,MAAc;IACtC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IACzC,OAAO,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAc;IACxC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,OAAO,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,gBAAgB;IAC7B,OAAO,aAAa,CAAC,gBAAgB,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,iBAAiB;IAC9B,OAAO,aAAa,CAAC,iBAAiB,CAAC;AACzC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,SAAS;IACtB,OAAO;QACL,QAAQ,EAAR,8BAAQ;QACR,aAAa,EAAE,aAAa,CAAC,gBAAgB;QAC7C,aAAa,EAAE,aAAa,CAAC,iBAAiB;QAC9C,UAAU,EAAE,aAAa,CAAC,UAAU;KACrC,CAAC;AACJ,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,aAAa;IAC1B,MAAM,SAAS,GAAG,IAAA,sBAAiB,GAAE,CAAC,eAAe,CAAC;IACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;IAEhD,OAAO,SAAS,GAAG,QAAQ,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,eAAuB;IAC3D,MAAM,IAAA,4BAAgB,EAAC,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED,wDAAwD;AACvD,MAAc,CAAC,IAAI,GAAG,sBAAU,CAAC,MAAM,CAItC,CAAC,GAAG,yBAAa,EAAE,GAAG,6BAAiB,EAAE,GAAG,4DAAyB,CAAC,EACtE;IACE,UAAU;IACV,UAAU;IACV,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,SAAS;IACT,aAAa;IACb,sBAAsB;CACvB,EACD,MAAM,CACP,CAAC;AA0BF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;IACpC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtC,MAAM,CAAC,CAAC;AACV,CAAC,CAAC,CAAC","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\n// initlogger and yargs must be imported before all other imports\n// making sure logger is defined before its called\n// eslint-disable-next-line import/order\nimport { initLogger } from '@subql/node-core/logger';\n// eslint-disable-next-line import/order\nimport { yargsOptions } from '../../yargs';\n\nconst { argv } = yargsOptions;\n\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\nimport assert from 'assert';\nimport { threadId } from 'node:worker_threads';\nimport { getHeapStatistics } from 'v8';\nimport { INestApplication } from '@nestjs/common';\nimport { NestFactory } from '@nestjs/core';\nimport {\n  waitForBatchSize,\n  WorkerHost,\n  getLogger,\n  NestLogger,\n  hostStoreKeys,\n  HostStore,\n  hostDynamicDsKeys,\n  HostDynamicDS,\n  ProcessBlockResponse,\n} from '@subql/node-core';\nimport { SubqlProjectDs } from '../../configure/SubqueryProject';\nimport { IndexerManager } from '../indexer.manager';\nimport { WorkerModule } from './worker.module';\nimport {\n  FetchBlockResponse,\n  WorkerService,\n  WorkerStatusResponse,\n} from './worker.service';\nimport {\n  HostUnfinalizedBlocks,\n  hostUnfinalizedBlocksKeys,\n} from './worker.unfinalizedBlocks.service';\nlet app: INestApplication;\nlet workerService: WorkerService;\n\nconst logger = getLogger(`worker #${threadId}`);\n\nasync function initWorker(): Promise<void> {\n  try {\n    if (app) {\n      logger.warn('Worker already initialised');\n      return;\n    }\n\n    app = await NestFactory.create(WorkerModule, {\n      logger: new NestLogger(), // TIP: If the worker is crashing comment out this line for better logging\n    });\n\n    await app.init();\n\n    const indexerManager = app.get(IndexerManager);\n    // Initialise async services, we do this here rather than in factories so we can capture one off events\n    await indexerManager.start();\n\n    workerService = app.get(WorkerService);\n  } catch (e) {\n    console.log('Failed to start worker', e);\n    logger.error(e, 'Failed to start worker');\n    throw e;\n  }\n}\n\nasync function fetchBlock(height: number): Promise<FetchBlockResponse> {\n  assert(workerService, 'Not initialised');\n  return workerService.fetchBlock(height);\n}\n\nasync function processBlock(height: number): Promise<ProcessBlockResponse> {\n  assert(workerService, 'Not initialised');\n\n  return workerService.processBlock(height);\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchedBlocks(): Promise<number> {\n  return workerService.numFetchedBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchingBlocks(): Promise<number> {\n  return workerService.numFetchingBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getStatus(): Promise<WorkerStatusResponse> {\n  return {\n    threadId,\n    fetchedBlocks: workerService.numFetchedBlocks,\n    toFetchBlocks: workerService.numFetchingBlocks,\n    isIndexing: workerService.isIndexing,\n  };\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getMemoryLeft(): Promise<number> {\n  const totalHeap = getHeapStatistics().heap_size_limit;\n  const heapUsed = process.memoryUsage().heapUsed;\n\n  return totalHeap - heapUsed;\n}\n\nasync function waitForWorkerBatchSize(heapSizeInBytes: number): Promise<void> {\n  await waitForBatchSize(heapSizeInBytes);\n}\n\n// Register these functions to be exposed to worker host\n(global as any).host = WorkerHost.create<\n  HostStore & HostDynamicDS<SubqlProjectDs> & HostUnfinalizedBlocks,\n  IInitIndexerWorker\n>(\n  [...hostStoreKeys, ...hostDynamicDsKeys, ...hostUnfinalizedBlocksKeys],\n  {\n    initWorker,\n    fetchBlock,\n    processBlock,\n    numFetchedBlocks,\n    numFetchingBlocks,\n    getStatus,\n    getMemoryLeft,\n    waitForWorkerBatchSize,\n  },\n  logger,\n);\n\n// Export types to be used on the parent\ntype InitWorker = typeof initWorker;\ntype FetchBlock = typeof fetchBlock;\ntype ProcessBlock = typeof processBlock;\ntype NumFetchedBlocks = typeof numFetchedBlocks;\ntype NumFetchingBlocks = typeof numFetchingBlocks;\ntype GetWorkerStatus = typeof getStatus;\ntype GetMemoryLeft = typeof getMemoryLeft;\ntype WaitForWorkerBatchSize = typeof waitForWorkerBatchSize;\n\nexport type IIndexerWorker = {\n  processBlock: ProcessBlock;\n  fetchBlock: FetchBlock;\n  numFetchedBlocks: NumFetchedBlocks;\n  numFetchingBlocks: NumFetchingBlocks;\n  getStatus: GetWorkerStatus;\n  getMemoryLeft: GetMemoryLeft;\n  waitForWorkerBatchSize: WaitForWorkerBatchSize;\n};\n\nexport type IInitIndexerWorker = IIndexerWorker & {\n  initWorker: InitWorker;\n};\n\nprocess.on('uncaughtException', (e) => {\n  logger.error(e, 'Uncaught Exception');\n  throw e;\n});\n"]}

package/dist/indexer/worker/worker.service.d.ts

@@ -1,15 +1,9 @@
-import { NodeConfig, IProjectService } from '@subql/node-core';
+import { NodeConfig, IProjectService, ProcessBlockResponse, ApiService } from '@subql/node-core';
 import { SubqlProjectDs } from '../../configure/SubqueryProject';
-import { EthereumApiService } from '../../ethereum';
 import { IndexerManager } from '../indexer.manager';
 export type FetchBlockResponse = {
     parentHash: string;
 } | undefined;
-export type ProcessBlockResponse = {
-    dynamicDsCreated: boolean;
-    blockHash: string;
-    reindexBlockHeight: number;
-};
 export type WorkerStatusResponse = {
     threadId: number;
     isIndexing: boolean;
@@ -23,7 +17,7 @@ export declare class WorkerService {
     private fetchedBlocks;
     private _isIndexing;
     private queue;
-    constructor(apiService: EthereumApiService, indexerManager: IndexerManager, projectService: IProjectService<SubqlProjectDs>, nodeConfig: NodeConfig);
+    constructor(apiService: ApiService, indexerManager: IndexerManager, projectService: IProjectService<SubqlProjectDs>, nodeConfig: NodeConfig);
     fetchBlock(height: number): Promise<FetchBlockResponse>;
     processBlock(height: number): Promise<ProcessBlockResponse>;
     get numFetchedBlocks(): number;

package/dist/indexer/worker/worker.service.js

@@ -18,7 +18,6 @@ exports.WorkerService = void 0;
 const node_worker_threads_1 = require("node:worker_threads");
 const common_1 = require("@nestjs/common");
 const node_core_1 = require("@subql/node-core");
-const ethereum_1 = require("../../ethereum");
 const indexer_manager_1 = require("../indexer.manager");
 const logger = (0, node_core_1.getLogger)(`Worker Service #${node_worker_threads_1.threadId}`);
 let WorkerService = class WorkerService {
@@ -44,7 +43,6 @@ let WorkerService = class WorkerService {
                     const [block] = await this.apiService.fetchBlocks([height]);
                     this.fetchedBlocks[height] = block;
                 }
-                // const block = this.fetchedBlocks[height];
                 // Return info to get the runtime version, this lets the worker thread know
                 return undefined;
             });
@@ -85,7 +83,7 @@ let WorkerService = class WorkerService {
 WorkerService = __decorate([
     (0, common_1.Injectable)(),
     __param(2, (0, common_1.Inject)('IProjectService')),
-    __metadata("design:paramtypes", [ethereum_1.EthereumApiService,
+    __metadata("design:paramtypes", [node_core_1.ApiService,
         indexer_manager_1.IndexerManager, Object, node_core_1.NodeConfig])
 ], WorkerService);
 exports.WorkerService = WorkerService;

package/dist/indexer/worker/worker.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,6DAA+C;AAC/C,2CAAoD;AACpD,gDAM0B;AAG1B,6CAAoD;AACpD,wDAAoD;AAiBpD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,8BAAQ,EAAE,CAAC,CAAC;AAGjD,IAAM,aAAa,GAAnB,MAAM,aAAa;IAMxB,YACU,UAA8B,EAC9B,cAA8B,EAE9B,cAA+C,EACvD,UAAsB;QAJd,eAAU,GAAV,UAAU,CAAoB;QAC9B,mBAAc,GAAd,cAAc,CAAgB;QAE9B,mBAAc,GAAd,cAAc,CAAiC;QATjD,kBAAa,GAAiC,EAAE,CAAC;QACjD,gBAAW,GAAG,KAAK,CAAC;QAW1B,IAAI,CAAC,KAAK,GAAG,IAAI,qBAAS,CAAC,SAAS,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACrC,0FAA0F;gBAC1F,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;oBAC/B,IAAI,sBAAU,CAAC,QAAQ,EAAE,EAAE;wBACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACzB,MAAM,sBAAU,CAAC,aAAa,EAAE,CAAC;wBACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACvB,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC;qBACzD;oBAED,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;oBAC5D,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;iBACpC;gBAED,4CAA4C;gBAC5C,2EAA2E;gBAC3E,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;SACJ;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,yBAAyB,MAAM,EAAE,CAAC,CAAC;YACxD,MAAM,CAAC,CAAC;SACT;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc;QAC/B,IAAI;YACF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAyB,CAAC;YAEjE,IAAI,CAAC,KAAK,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,uBAAuB,CAAC,CAAC;aACzD;YAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAElC,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CACzC,KAAK,EACL,MAAM,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CACpD,CAAC;SACH;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC1B;IACH,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF,CAAA;AA5EY,aAAa;IADzB,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;qCAFN,6BAAkB;QACd,gCAAc,UAG1B,sBAAU;GAXb,aAAa,CA4EzB;AA5EY,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { threadId } from 'node:worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  NodeConfig,\n  getLogger,\n  AutoQueue,\n  memoryLock,\n  IProjectService,\n} from '@subql/node-core';\nimport { BlockWrapper, EthereumBlockWrapper } from '@subql/types-ethereum';\nimport { SubqlProjectDs } from '../../configure/SubqueryProject';\nimport { EthereumApiService } from '../../ethereum';\nimport { IndexerManager } from '../indexer.manager';\n\nexport type FetchBlockResponse = { parentHash: string } | undefined;\n\nexport type ProcessBlockResponse = {\n  dynamicDsCreated: boolean;\n  blockHash: string;\n  reindexBlockHeight: number;\n};\n\nexport type WorkerStatusResponse = {\n  threadId: number;\n  isIndexing: boolean;\n  fetchedBlocks: number;\n  toFetchBlocks: number;\n};\n\nconst logger = getLogger(`Worker Service #${threadId}`);\n\n@Injectable()\nexport class WorkerService {\n  private fetchedBlocks: Record<string, BlockWrapper> = {};\n  private _isIndexing = false;\n\n  private queue: AutoQueue<FetchBlockResponse>;\n\n  constructor(\n    private apiService: EthereumApiService,\n    private indexerManager: IndexerManager,\n    @Inject('IProjectService')\n    private projectService: IProjectService<SubqlProjectDs>,\n    nodeConfig: NodeConfig,\n  ) {\n    this.queue = new AutoQueue(undefined, nodeConfig.batchSize);\n  }\n\n  async fetchBlock(height: number): Promise<FetchBlockResponse> {\n    try {\n      return await this.queue.put(async () => {\n        // If a dynamic ds is created we might be asked to fetch blocks again, use existing result\n        if (!this.fetchedBlocks[height]) {\n          if (memoryLock.isLocked()) {\n            const start = Date.now();\n            await memoryLock.waitForUnlock();\n            const end = Date.now();\n            logger.debug(`memory lock wait time: ${end - start}ms`);\n          }\n\n          const [block] = await this.apiService.fetchBlocks([height]);\n          this.fetchedBlocks[height] = block;\n        }\n\n        // const block = this.fetchedBlocks[height];\n        // Return info to get the runtime version, this lets the worker thread know\n        return undefined;\n      });\n    } catch (e) {\n      logger.error(/*e, */ `Failed to fetch block ${height}`);\n      throw e;\n    }\n  }\n\n  async processBlock(height: number): Promise<ProcessBlockResponse> {\n    try {\n      this._isIndexing = true;\n      const block = this.fetchedBlocks[height] as EthereumBlockWrapper;\n\n      if (!block) {\n        throw new Error(`Block ${height} has not been fetched`);\n      }\n\n      delete this.fetchedBlocks[height];\n\n      return await this.indexerManager.indexBlock(\n        block,\n        await this.projectService.getAllDataSources(height),\n      );\n    } catch (e) {\n      logger.error(e, `Failed to index block ${height}: ${e.stack}`);\n      throw e;\n    } finally {\n      this._isIndexing = false;\n    }\n  }\n\n  get numFetchedBlocks(): number {\n    return Object.keys(this.fetchedBlocks).length;\n  }\n\n  get numFetchingBlocks(): number {\n    return this.queue.size;\n  }\n\n  get isIndexing(): boolean {\n    return this._isIndexing;\n  }\n}\n"]}
+{"version":3,"file":"worker.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,6DAA+C;AAC/C,2CAAoD;AACpD,gDAQ0B;AAG1B,wDAAoD;AAWpD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,8BAAQ,EAAE,CAAC,CAAC;AAGjD,IAAM,aAAa,GAAnB,MAAM,aAAa;IAMxB,YACU,UAAsB,EACtB,cAA8B,EAE9B,cAA+C,EACvD,UAAsB;QAJd,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;QAE9B,mBAAc,GAAd,cAAc,CAAiC;QATjD,kBAAa,GAAiC,EAAE,CAAC;QACjD,gBAAW,GAAG,KAAK,CAAC;QAW1B,IAAI,CAAC,KAAK,GAAG,IAAI,qBAAS,CAAC,SAAS,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACrC,0FAA0F;gBAC1F,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;oBAC/B,IAAI,sBAAU,CAAC,QAAQ,EAAE,EAAE;wBACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACzB,MAAM,sBAAU,CAAC,aAAa,EAAE,CAAC;wBACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACvB,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC;qBACzD;oBAED,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;oBAC5D,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;iBACpC;gBAED,2EAA2E;gBAC3E,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;SACJ;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,yBAAyB,MAAM,EAAE,CAAC,CAAC;YACxD,MAAM,CAAC,CAAC;SACT;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc;QAC/B,IAAI;YACF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAyB,CAAC;YAEjE,IAAI,CAAC,KAAK,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,uBAAuB,CAAC,CAAC;aACzD;YAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAElC,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CACzC,KAAK,EACL,MAAM,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CACpD,CAAC;SACH;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC1B;IACH,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF,CAAA;AA3EY,aAAa;IADzB,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;qCAFN,sBAAU;QACN,gCAAc,UAG1B,sBAAU;GAXb,aAAa,CA2EzB;AA3EY,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { threadId } from 'node:worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  NodeConfig,\n  getLogger,\n  AutoQueue,\n  memoryLock,\n  IProjectService,\n  ProcessBlockResponse,\n  ApiService,\n} from '@subql/node-core';\nimport { BlockWrapper, EthereumBlockWrapper } from '@subql/types-ethereum';\nimport { SubqlProjectDs } from '../../configure/SubqueryProject';\nimport { IndexerManager } from '../indexer.manager';\n\nexport type FetchBlockResponse = { parentHash: string } | undefined;\n\nexport type WorkerStatusResponse = {\n  threadId: number;\n  isIndexing: boolean;\n  fetchedBlocks: number;\n  toFetchBlocks: number;\n};\n\nconst logger = getLogger(`Worker Service #${threadId}`);\n\n@Injectable()\nexport class WorkerService {\n  private fetchedBlocks: Record<string, BlockWrapper> = {};\n  private _isIndexing = false;\n\n  private queue: AutoQueue<FetchBlockResponse>;\n\n  constructor(\n    private apiService: ApiService,\n    private indexerManager: IndexerManager,\n    @Inject('IProjectService')\n    private projectService: IProjectService<SubqlProjectDs>,\n    nodeConfig: NodeConfig,\n  ) {\n    this.queue = new AutoQueue(undefined, nodeConfig.batchSize);\n  }\n\n  async fetchBlock(height: number): Promise<FetchBlockResponse> {\n    try {\n      return await this.queue.put(async () => {\n        // If a dynamic ds is created we might be asked to fetch blocks again, use existing result\n        if (!this.fetchedBlocks[height]) {\n          if (memoryLock.isLocked()) {\n            const start = Date.now();\n            await memoryLock.waitForUnlock();\n            const end = Date.now();\n            logger.debug(`memory lock wait time: ${end - start}ms`);\n          }\n\n          const [block] = await this.apiService.fetchBlocks([height]);\n          this.fetchedBlocks[height] = block;\n        }\n\n        // Return info to get the runtime version, this lets the worker thread know\n        return undefined;\n      });\n    } catch (e) {\n      logger.error(/*e, */ `Failed to fetch block ${height}`);\n      throw e;\n    }\n  }\n\n  async processBlock(height: number): Promise<ProcessBlockResponse> {\n    try {\n      this._isIndexing = true;\n      const block = this.fetchedBlocks[height] as EthereumBlockWrapper;\n\n      if (!block) {\n        throw new Error(`Block ${height} has not been fetched`);\n      }\n\n      delete this.fetchedBlocks[height];\n\n      return await this.indexerManager.indexBlock(\n        block,\n        await this.projectService.getAllDataSources(height),\n      );\n    } catch (e) {\n      logger.error(e, `Failed to index block ${height}: ${e.stack}`);\n      throw e;\n    } finally {\n      this._isIndexing = false;\n    }\n  }\n\n  get numFetchedBlocks(): number {\n    return Object.keys(this.fetchedBlocks).length;\n  }\n\n  get numFetchingBlocks(): number {\n    return this.queue.size;\n  }\n\n  get isIndexing(): boolean {\n    return this._isIndexing;\n  }\n}\n"]}

package/dist/indexer/worker/worker.unfinalizedBlocks.service.d.ts

@@ -1,11 +1,14 @@
-import { EthereumBlock } from '@subql/types-ethereum';
-import { IUnfinalizedBlocksService } from '../unfinalizedBlocks.service';
+import { IUnfinalizedBlocksService } from '@subql/node-core';
+import { BlockWrapper } from '@subql/types-ethereum';
 export type HostUnfinalizedBlocks = {
-    unfinalizedBlocksProcess: (block: EthereumBlock) => Promise<number | null>;
+    unfinalizedBlocksProcess: (block: BlockWrapper) => Promise<number | null>;
 };
 export declare const hostUnfinalizedBlocksKeys: (keyof HostUnfinalizedBlocks)[];
-export declare class WorkerUnfinalizedBlocksService implements IUnfinalizedBlocksService {
+export declare class WorkerUnfinalizedBlocksService implements IUnfinalizedBlocksService<BlockWrapper> {
     private host;
     constructor(host: HostUnfinalizedBlocks);
-    processUnfinalizedBlocks(block: EthereumBlock): Promise<number | null>;
+    processUnfinalizedBlocks(block: BlockWrapper): Promise<number | null>;
+    init(reindex: (targetHeight: number) => Promise<void>): Promise<number>;
+    resetUnfinalizedBlocks(): void;
+    resetLastFinalizedVerifiedHeight(): void;
 }