@subql/node-stellar 2.9.3-2

package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/worker-block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,8DAA8D;AAC9D,mCAAmC;;;;;;;;;;;;;;;;;;AAEnC,gDAAwB;AACxB,2CAA2E;AAC3E,yDAAsD;AACtD,gDAiB0B;AAG1B,qEAGyC;AAGzC,8DAAyD;AACzD,4EAAwE;AAIxE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,8BAA8B,CAAC,CAAC;AAMzD,KAAK,UAAU,mBAAmB,CAChC,KAAY,EACZ,gBAAmD,EACnD,wBAAwE,EACxE,mBAAqE,EACrE,IAAY;IAEZ,MAAM,aAAa,GAAG,kBAAM,CAAC,MAAM,CAIjC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,EACjE;QACE,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,mBAAmB;QACnB,WAAW;QACX,eAAe;QACf,wBAAwB;KACzB,kBAEC,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAC/B,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAC7C,kBAAkB,EAAE,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,EACnD,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAC/B,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAC7C,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAC7C,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EACrC,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAC7C,gCAAgC,EAC9B,gBAAgB,CAAC,uBAAuB,CAAC,IAAI,CAAC,gBAAgB,CAAC,EACjE,8BAA8B,EAC5B,gBAAgB,CAAC,qBAAqB,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAC/D,wBAAwB,EACtB,wBAAwB,CAAC,6BAA6B,CAAC,IAAI,CACzD,wBAAwB,CACzB,IACA,IAAA,4CAAgC,EAAC,mBAAmB,CAAC,GAE1D,IAAI,CACL,CAAC;IAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;IAEjC,OAAO,aAAa,CAAC;AACvB,CAAC;AAGM,IAAM,4BAA4B,GAAlC,MAAM,4BACX,SAAQ,iCAAoD;IAG5D,YACE,UAAsB,EACtB,YAA2B,EACA,cAA+C,EAC1E,iBAAoC,EACpC,YAA0B,EAC1B,iBAAoC,EACpC,UAAsB,EACM,OAAwB,EACpD,gBAAkC,EAClC,uBAAiD,EACjD,mBAAqE;QAErE,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,OAAO,EACP,gBAAgB,EAChB,GAAG,EAAE,CACH,mBAAmB,CACjB,YAAY,CAAC,QAAQ,EAAE,EACvB,gBAAgB,EAChB,uBAAuB,EACvB,mBAAmB,EACnB,OAAO,CAAC,IAAI,CACb,CACJ,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,UAAU,CACxB,MAAqB,EACrB,MAAc;QAEd,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,oDAAoD;QACpD,yBAAyB;QACzB,iBAAiB;QACjB,yEAAyE;QACzE,OAAO;QACP,+BAA+B;QAC/B,iBAAiB;QACjB,4EAA4E;QAC5E,OAAO;QACP,IAAI;IACN,CAAC;CACF,CAAA;AAzDY,4BAA4B;IADxC,IAAA,mBAAU,GAAE;IAQR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;IAKzB,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAPf,sBAAU;QACR,6BAAa,UAER,6BAAiB;QACtB,wBAAY;QACP,6BAAiB;QACxB,sBAAU;QACe,iCAAe;QAClC,qCAAgB;QACT,oDAAwB;QAC5B,sCAA0B;GAftC,4BAA4B,CAyDxC;AAzDY,oEAA4B","sourcesContent":["// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors\n// SPDX-License-Identifier: GPL-3.0\n\nimport path from 'path';\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  getLogger,\n  NodeConfig,\n  Worker,\n  SmartBatchService,\n  StoreService,\n  PoiService,\n  StoreCacheService,\n  IProjectService,\n  IDynamicDsService,\n  HostStore,\n  HostDynamicDS,\n  WorkerBlockDispatcher,\n  IUnfinalizedBlocksService,\n  HostConnectionPoolState,\n  ConnectionPoolStateManager,\n  connectionPoolStateHostFunctions,\n} from '@subql/node-core';\nimport { Store } from '@subql/types-stellar';\nimport chalk from 'chalk';\nimport {\n  SubqlProjectDs,\n  SubqueryProject,\n} from '../../configure/SubqueryProject';\nimport { StellarApiConnection } from '../../stellar/api.connection';\nimport { StellarBlockWrapped } from '../../stellar/block.stellar';\nimport { DynamicDsService } from '../dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../unfinalizedBlocks.service';\nimport { IIndexerWorker, IInitIndexerWorker } from '../worker/worker';\nimport { HostUnfinalizedBlocks } from '../worker/worker.unfinalizedBlocks.service';\n\nconst logger = getLogger('WorkerBlockDispatcherService');\n\ntype IndexerWorker = IIndexerWorker & {\n  terminate: () => Promise<number>;\n};\n\nasync function createIndexerWorker(\n  store: Store,\n  dynamicDsService: IDynamicDsService<SubqlProjectDs>,\n  unfinalizedBlocksService: IUnfinalizedBlocksService<StellarBlockWrapped>,\n  connectionPoolState: ConnectionPoolStateManager<StellarApiConnection>,\n  root: string,\n): Promise<IndexerWorker> {\n  const indexerWorker = Worker.create<\n    IInitIndexerWorker,\n    HostDynamicDS<SubqlProjectDs> & HostStore & HostUnfinalizedBlocks\n  >(\n    path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),\n    [\n      'initWorker',\n      'processBlock',\n      'fetchBlock',\n      'numFetchedBlocks',\n      'numFetchingBlocks',\n      'getStatus',\n      'getMemoryLeft',\n      'waitForWorkerBatchSize',\n    ],\n    {\n      storeGet: store.get.bind(store),\n      storeGetByField: store.getByField.bind(store),\n      storeGetOneByField: store.getOneByField.bind(store),\n      storeSet: store.set.bind(store),\n      storeBulkCreate: store.bulkCreate.bind(store),\n      storeBulkUpdate: store.bulkUpdate.bind(store),\n      storeRemove: store.remove.bind(store),\n      storeBulkRemove: store.bulkRemove.bind(store),\n      dynamicDsCreateDynamicDatasource:\n        dynamicDsService.createDynamicDatasource.bind(dynamicDsService),\n      dynamicDsGetDynamicDatasources:\n        dynamicDsService.getDynamicDatasources.bind(dynamicDsService),\n      unfinalizedBlocksProcess:\n        unfinalizedBlocksService.processUnfinalizedBlockHeader.bind(\n          unfinalizedBlocksService,\n        ),\n      ...connectionPoolStateHostFunctions(connectionPoolState),\n    },\n    root,\n  );\n\n  await indexerWorker.initWorker();\n\n  return indexerWorker;\n}\n\n@Injectable()\nexport class WorkerBlockDispatcherService\n  extends WorkerBlockDispatcher<SubqlProjectDs, IndexerWorker>\n  implements OnApplicationShutdown\n{\n  constructor(\n    nodeConfig: NodeConfig,\n    eventEmitter: EventEmitter2,\n    @Inject('IProjectService') projectService: IProjectService<SubqlProjectDs>,\n    smartBatchService: SmartBatchService,\n    storeService: StoreService,\n    storeCacheService: StoreCacheService,\n    poiService: PoiService,\n    @Inject('ISubqueryProject') project: SubqueryProject,\n    dynamicDsService: DynamicDsService,\n    unfinalizedBlocksSevice: UnfinalizedBlocksService,\n    connectionPoolState: ConnectionPoolStateManager<StellarApiConnection>,\n  ) {\n    super(\n      nodeConfig,\n      eventEmitter,\n      projectService,\n      smartBatchService,\n      storeService,\n      storeCacheService,\n      poiService,\n      project,\n      dynamicDsService,\n      () =>\n        createIndexerWorker(\n          storeService.getStore(),\n          dynamicDsService,\n          unfinalizedBlocksSevice,\n          connectionPoolState,\n          project.root,\n        ),\n    );\n  }\n\n  protected async fetchBlock(\n    worker: IndexerWorker,\n    height: number,\n  ): Promise<void> {\n    const start = new Date();\n    await worker.fetchBlock(height);\n    const end = new Date();\n\n    // const waitTime = end.getTime() - start.getTime();\n    // if (waitTime > 1000) {\n    //   logger.info(\n    //     `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,\n    //   );\n    // } else if (waitTime > 200) {\n    //   logger.info(\n    //     `Waiting to fetch block ${height}: ${chalk.yellow(`${waitTime}ms`)}`,\n    //   );\n    // }\n  }\n}\n"]}

package/dist/indexer/dictionary.service.d.ts

@@ -0,0 +1,7 @@
+import { OnApplicationShutdown } from '@nestjs/common';
+import { NodeConfig, DictionaryService as CoreDictionaryService } from '@subql/node-core';
+import { SubqueryProject } from '../configure/SubqueryProject';
+export declare class DictionaryService extends CoreDictionaryService implements OnApplicationShutdown {
+    protected project: SubqueryProject;
+    constructor(project: SubqueryProject, nodeConfig: NodeConfig);
+}

package/dist/indexer/dictionary.service.js

@@ -0,0 +1,34 @@
+"use strict";
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DictionaryService = void 0;
+const common_1 = require("@nestjs/common");
+const node_core_1 = require("@subql/node-core");
+const SubqueryProject_1 = require("../configure/SubqueryProject");
+let DictionaryService = class DictionaryService extends node_core_1.DictionaryService {
+    constructor(project, nodeConfig) {
+        super(project.network.dictionary, project.network.chainId, nodeConfig);
+        this.project = project;
+    }
+};
+DictionaryService = __decorate([
+    (0, common_1.Injectable)(),
+    __param(0, (0, common_1.Inject)('ISubqueryProject')),
+    __metadata("design:paramtypes", [SubqueryProject_1.SubqueryProject,
+        node_core_1.NodeConfig])
+], DictionaryService);
+exports.DictionaryService = DictionaryService;
+//# sourceMappingURL=dictionary.service.js.map

package/dist/indexer/dictionary.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dictionary.service.js","sourceRoot":"","sources":["../../src/indexer/dictionary.service.ts"],"names":[],"mappings":";AAAA,8DAA8D;AAC9D,mCAAmC;;;;;;;;;;;;;;;AAEnC,2CAA2E;AAC3E,gDAG0B;AAC1B,kEAA+D;AAGxD,IAAM,iBAAiB,GAAvB,MAAM,iBACX,SAAQ,6BAAqB;IAG7B,YACwC,OAAwB,EAC9D,UAAsB;QAEtB,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAHjC,YAAO,GAAP,OAAO,CAAiB;IAIhE,CAAC;CACF,CAAA;AAVY,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;IAMR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAAoB,iCAAe;QAClD,sBAAU;GANb,iBAAiB,CAU7B;AAVY,8CAAiB","sourcesContent":["// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors\n// SPDX-License-Identifier: GPL-3.0\n\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport {\n  NodeConfig,\n  DictionaryService as CoreDictionaryService,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\n@Injectable()\nexport class DictionaryService\n  extends CoreDictionaryService\n  implements OnApplicationShutdown\n{\n  constructor(\n    @Inject('ISubqueryProject') protected project: SubqueryProject,\n    nodeConfig: NodeConfig,\n  ) {\n    super(project.network.dictionary, project.network.chainId, nodeConfig);\n  }\n}\n"]}

package/dist/indexer/ds-processor.service.d.ts

@@ -0,0 +1,9 @@
+import { StellarHandlerKind, isCustomDs, SubqlStellarCustomDataSource, SubqlStellarDataSource, SubqlDatasourceProcessor } from '@subql/common-stellar';
+import { BaseDsProcessorService } from '@subql/node-core';
+import { SecondLayerHandlerProcessor_0_0_0, SecondLayerHandlerProcessor_1_0_0, SubqlCustomDatasource } from '@subql/types-stellar';
+export declare function isSecondLayerHandlerProcessor_0_0_0<K extends StellarHandlerKind, F, E, DS extends SubqlCustomDatasource = SubqlStellarCustomDataSource>(processor: SecondLayerHandlerProcessor_0_0_0<K, F, E, DS> | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>): processor is SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>;
+export declare function isSecondLayerHandlerProcessor_1_0_0<K extends StellarHandlerKind, F, E, DS extends SubqlStellarCustomDataSource = SubqlStellarCustomDataSource>(processor: SecondLayerHandlerProcessor_0_0_0<K, F, E, DS> | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>): processor is SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>;
+export declare function asSecondLayerHandlerProcessor_1_0_0<K extends StellarHandlerKind, F, E, DS extends SubqlStellarCustomDataSource = SubqlStellarCustomDataSource>(processor: SecondLayerHandlerProcessor_0_0_0<K, F, E, DS> | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>): SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>;
+export declare class DsProcessorService extends BaseDsProcessorService<SubqlStellarDataSource, SubqlStellarCustomDataSource<string>, SubqlDatasourceProcessor<string, unknown>> {
+    protected isCustomDs: typeof isCustomDs;
+}

package/dist/indexer/ds-processor.service.js

@@ -0,0 +1,46 @@
+"use strict";
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DsProcessorService = exports.asSecondLayerHandlerProcessor_1_0_0 = exports.isSecondLayerHandlerProcessor_1_0_0 = exports.isSecondLayerHandlerProcessor_0_0_0 = void 0;
+const common_1 = require("@nestjs/common");
+const common_stellar_1 = require("@subql/common-stellar");
+const node_core_1 = require("@subql/node-core");
+function isSecondLayerHandlerProcessor_0_0_0(processor) {
+    // Exisiting datasource processors had no concept of specVersion, therefore undefined is equivalent to 0.0.0
+    return processor.specVersion === undefined;
+}
+exports.isSecondLayerHandlerProcessor_0_0_0 = isSecondLayerHandlerProcessor_0_0_0;
+function isSecondLayerHandlerProcessor_1_0_0(processor) {
+    return processor.specVersion === '1.0.0';
+}
+exports.isSecondLayerHandlerProcessor_1_0_0 = isSecondLayerHandlerProcessor_1_0_0;
+function asSecondLayerHandlerProcessor_1_0_0(processor) {
+    if (isSecondLayerHandlerProcessor_1_0_0(processor)) {
+        return processor;
+    }
+    if (!isSecondLayerHandlerProcessor_0_0_0(processor)) {
+        throw new Error('Unsupported ds processor version');
+    }
+    return Object.assign(Object.assign({}, processor), { specVersion: '1.0.0', filterProcessor: (params) => processor.filterProcessor(params.filter, params.input, params.ds), transformer: (params) => processor
+            .transformer(params.input, params.ds, params.api, params.assets)
+            .then((res) => [res]) });
+}
+exports.asSecondLayerHandlerProcessor_1_0_0 = asSecondLayerHandlerProcessor_1_0_0;
+let DsProcessorService = class DsProcessorService extends node_core_1.BaseDsProcessorService {
+    constructor() {
+        super(...arguments);
+        this.isCustomDs = common_stellar_1.isCustomDs;
+    }
+};
+DsProcessorService = __decorate([
+    (0, common_1.Injectable)()
+], DsProcessorService);
+exports.DsProcessorService = DsProcessorService;
+//# sourceMappingURL=ds-processor.service.js.map

package/dist/indexer/ds-processor.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ds-processor.service.js","sourceRoot":"","sources":["../../src/indexer/ds-processor.service.ts"],"names":[],"mappings":";AAAA,8DAA8D;AAC9D,mCAAmC;;;;;;;;;AAEnC,2CAA4C;AAC5C,0DAM+B;AAC/B,gDAA0D;AAO1D,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,4GAA4G;IAC5G,OAAO,SAAS,CAAC,WAAW,KAAK,SAAS,CAAC;AAC7C,CAAC;AAZD,kFAYC;AAED,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,OAAO,SAAS,CAAC,WAAW,KAAK,OAAO,CAAC;AAC3C,CAAC;AAXD,kFAWC;AAED,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,IAAI,mCAAmC,CAAC,SAAS,CAAC,EAAE;QAClD,OAAO,SAAS,CAAC;KAClB;IAED,IAAI,CAAC,mCAAmC,CAAC,SAAS,CAAC,EAAE;QACnD,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;KACrD;IAED,uCACK,SAAS,KACZ,WAAW,EAAE,OAAO,EACpB,eAAe,EAAE,CAAC,MAAM,EAAE,EAAE,CAC1B,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,EACnE,WAAW,EAAE,CAAC,MAAM,EAAE,EAAE,CACtB,SAAS;aACN,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC;aAC/D,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IACzB;AACJ,CAAC;AA5BD,kFA4BC;AAGM,IAAM,kBAAkB,GAAxB,MAAM,kBAAmB,SAAQ,kCAIvC;IAJM;;QAKK,eAAU,GAAG,2BAAU,CAAC;IACpC,CAAC;CAAA,CAAA;AANY,kBAAkB;IAD9B,IAAA,mBAAU,GAAE;GACA,kBAAkB,CAM9B;AANY,gDAAkB","sourcesContent":["// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors\n// SPDX-License-Identifier: GPL-3.0\n\nimport { Injectable } from '@nestjs/common';\nimport {\n  StellarHandlerKind,\n  isCustomDs,\n  SubqlStellarCustomDataSource,\n  SubqlStellarDataSource,\n  SubqlDatasourceProcessor,\n} from '@subql/common-stellar';\nimport { BaseDsProcessorService } from '@subql/node-core';\nimport {\n  SecondLayerHandlerProcessor_0_0_0,\n  SecondLayerHandlerProcessor_1_0_0,\n  SubqlCustomDatasource,\n} from '@subql/types-stellar';\n\nexport function isSecondLayerHandlerProcessor_0_0_0<\n  K extends StellarHandlerKind,\n  F,\n  E,\n  DS extends SubqlCustomDatasource = SubqlStellarCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): processor is SecondLayerHandlerProcessor_0_0_0<K, F, E, DS> {\n  // Exisiting datasource processors had no concept of specVersion, therefore undefined is equivalent to 0.0.0\n  return processor.specVersion === undefined;\n}\n\nexport function isSecondLayerHandlerProcessor_1_0_0<\n  K extends StellarHandlerKind,\n  F,\n  E,\n  DS extends SubqlStellarCustomDataSource = SubqlStellarCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): processor is SecondLayerHandlerProcessor_1_0_0<K, F, E, DS> {\n  return processor.specVersion === '1.0.0';\n}\n\nexport function asSecondLayerHandlerProcessor_1_0_0<\n  K extends StellarHandlerKind,\n  F,\n  E,\n  DS extends SubqlStellarCustomDataSource = SubqlStellarCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): SecondLayerHandlerProcessor_1_0_0<K, F, E, DS> {\n  if (isSecondLayerHandlerProcessor_1_0_0(processor)) {\n    return processor;\n  }\n\n  if (!isSecondLayerHandlerProcessor_0_0_0(processor)) {\n    throw new Error('Unsupported ds processor version');\n  }\n\n  return {\n    ...processor,\n    specVersion: '1.0.0',\n    filterProcessor: (params) =>\n      processor.filterProcessor(params.filter, params.input, params.ds),\n    transformer: (params) =>\n      processor\n        .transformer(params.input, params.ds, params.api, params.assets)\n        .then((res) => [res]),\n  };\n}\n\n@Injectable()\nexport class DsProcessorService extends BaseDsProcessorService<\n  SubqlStellarDataSource,\n  SubqlStellarCustomDataSource<string>,\n  SubqlDatasourceProcessor<string, unknown>\n> {\n  protected isCustomDs = isCustomDs;\n}\n"]}

package/dist/indexer/dynamic-ds.service.d.ts

@@ -0,0 +1,9 @@
+import { DatasourceParams, DynamicDsService as BaseDynamicDsService } from '@subql/node-core';
+import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';
+import { DsProcessorService } from './ds-processor.service';
+export declare class DynamicDsService extends BaseDynamicDsService<SubqlProjectDs> {
+    private readonly dsProcessorService;
+    private readonly project;
+    constructor(dsProcessorService: DsProcessorService, project: SubqueryProject);
+    protected getDatasource(params: DatasourceParams): Promise<SubqlProjectDs>;
+}

package/dist/indexer/dynamic-ds.service.js

@@ -0,0 +1,71 @@
+"use strict";
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DynamicDsService = void 0;
+const common_1 = require("@nestjs/common");
+const common_stellar_1 = require("@subql/common-stellar");
+const node_core_1 = require("@subql/node-core");
+const class_transformer_1 = require("class-transformer");
+const class_validator_1 = require("class-validator");
+const lodash_1 = require("lodash");
+const SubqueryProject_1 = require("../configure/SubqueryProject");
+const ds_processor_service_1 = require("./ds-processor.service");
+let DynamicDsService = class DynamicDsService extends node_core_1.DynamicDsService {
+    constructor(dsProcessorService, project) {
+        super();
+        this.dsProcessorService = dsProcessorService;
+        this.project = project;
+    }
+    async getDatasource(params) {
+        var _a;
+        const template = (0, lodash_1.cloneDeep)((_a = this.project.templates) === null || _a === void 0 ? void 0 : _a.find((t) => t.name === params.templateName));
+        if (!template) {
+            throw new Error(`Unable to find matching template in project for name: "${params.templateName}"`);
+        }
+        const dsObj = Object.assign(Object.assign({}, template), { startBlock: params.startBlock });
+        try {
+            if ((0, common_stellar_1.isCustomDs)(dsObj)) {
+                dsObj.processor.options = Object.assign(Object.assign({}, dsObj.processor.options), params.args);
+                await this.dsProcessorService.validateCustomDs([dsObj]);
+            }
+            else if ((0, common_stellar_1.isRuntimeDs)(dsObj)) {
+                dsObj.options = Object.assign(Object.assign({}, dsObj.options), params.args);
+                const parsedDs = (0, class_transformer_1.plainToClass)(common_stellar_1.StellarRuntimeDataSourceImpl, dsObj);
+                const errors = (0, class_validator_1.validateSync)(parsedDs, {
+                    whitelist: true,
+                    forbidNonWhitelisted: false,
+                });
+                if (errors.length) {
+                    throw new Error(`Dynamic ds is invalid\n${errors
+                        .map((e) => e.toString())
+                        .join('\n')}`);
+                }
+            }
+            return dsObj;
+        }
+        catch (e) {
+            throw new Error(`Unable to create dynamic datasource.\n ${e.message}`);
+        }
+    }
+};
+DynamicDsService = __decorate([
+    (0, common_1.Injectable)(),
+    __param(1, (0, common_1.Inject)('ISubqueryProject')),
+    __metadata("design:paramtypes", [ds_processor_service_1.DsProcessorService,
+        SubqueryProject_1.SubqueryProject])
+], DynamicDsService);
+exports.DynamicDsService = DynamicDsService;
+//# sourceMappingURL=dynamic-ds.service.js.map

package/dist/indexer/dynamic-ds.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dynamic-ds.service.js","sourceRoot":"","sources":["../../src/indexer/dynamic-ds.service.ts"],"names":[],"mappings":";AAAA,8DAA8D;AAC9D,mCAAmC;;;;;;;;;;;;;;;AAEnC,2CAAoD;AACpD,0DAI+B;AAC/B,gDAG0B;AAC1B,yDAAiD;AACjD,qDAA+C;AAC/C,mCAAmC;AACnC,kEAA+E;AAC/E,iEAA4D;AAGrD,IAAM,gBAAgB,GAAtB,MAAM,gBAAiB,SAAQ,4BAAoC;IACxE,YACmB,kBAAsC,EACV,OAAwB;QAErE,KAAK,EAAE,CAAC;QAHS,uBAAkB,GAAlB,kBAAkB,CAAoB;QACV,YAAO,GAAP,OAAO,CAAiB;IAGvE,CAAC;IAES,KAAK,CAAC,aAAa,CAC3B,MAAwB;;QAExB,MAAM,QAAQ,GAAG,IAAA,kBAAS,EACxB,MAAA,IAAI,CAAC,OAAO,CAAC,SAAS,0CAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,YAAY,CAAC,CACpE,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,KAAK,CACb,0DAA0D,MAAM,CAAC,YAAY,GAAG,CACjF,CAAC;SACH;QAED,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,UAAU,EAAE,MAAM,CAAC,UAAU,GACZ,CAAC;QACpB,IAAI;YACF,IAAI,IAAA,2BAAU,EAAC,KAAK,CAAC,EAAE;gBACrB,KAAK,CAAC,SAAS,CAAC,OAAO,mCAClB,KAAK,CAAC,SAAS,CAAC,OAAO,GACvB,MAAM,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACzD;iBAAM,IAAI,IAAA,4BAAW,EAAC,KAAK,CAAC,EAAE;gBAC7B,KAAK,CAAC,OAAO,mCACR,KAAK,CAAC,OAAO,GACb,MAAM,CAAC,IAAI,CACf,CAAC;gBAEF,MAAM,QAAQ,GAAG,IAAA,gCAAY,EAAC,6CAA4B,EAAE,KAAK,CAAC,CAAC;gBAEnE,MAAM,MAAM,GAAG,IAAA,8BAAY,EAAC,QAAQ,EAAE;oBACpC,SAAS,EAAE,IAAI;oBACf,oBAAoB,EAAE,KAAK;iBAC5B,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,EAAE;oBACjB,MAAM,IAAI,KAAK,CACb,0BAA0B,MAAM;yBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;yBACxB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;iBACH;aACF;YACD,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACxE;IACH,CAAC;CACF,CAAA;AAzDY,gBAAgB;IAD5B,IAAA,mBAAU,GAAE;IAIR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCADU,yCAAkB;QACD,iCAAe;GAH5D,gBAAgB,CAyD5B;AAzDY,4CAAgB","sourcesContent":["// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors\n// SPDX-License-Identifier: GPL-3.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  StellarRuntimeDataSourceImpl,\n  isCustomDs,\n  isRuntimeDs,\n} from '@subql/common-stellar';\nimport {\n  DatasourceParams,\n  DynamicDsService as BaseDynamicDsService,\n} from '@subql/node-core';\nimport { plainToClass } from 'class-transformer';\nimport { validateSync } from 'class-validator';\nimport { cloneDeep } from 'lodash';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { DsProcessorService } from './ds-processor.service';\n\n@Injectable()\nexport class DynamicDsService extends BaseDynamicDsService<SubqlProjectDs> {\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n  ) {\n    super();\n  }\n\n  protected async getDatasource(\n    params: DatasourceParams,\n  ): Promise<SubqlProjectDs> {\n    const template = cloneDeep(\n      this.project.templates?.find((t) => t.name === params.templateName),\n    );\n\n    if (!template) {\n      throw new Error(\n        `Unable to find matching template in project for name: \"${params.templateName}\"`,\n      );\n    }\n\n    const dsObj = {\n      ...template,\n      startBlock: params.startBlock,\n    } as SubqlProjectDs;\n    try {\n      if (isCustomDs(dsObj)) {\n        dsObj.processor.options = {\n          ...dsObj.processor.options,\n          ...params.args,\n        };\n        await this.dsProcessorService.validateCustomDs([dsObj]);\n      } else if (isRuntimeDs(dsObj)) {\n        dsObj.options = {\n          ...dsObj.options,\n          ...params.args,\n        };\n\n        const parsedDs = plainToClass(StellarRuntimeDataSourceImpl, dsObj);\n\n        const errors = validateSync(parsedDs, {\n          whitelist: true,\n          forbidNonWhitelisted: false,\n        });\n        if (errors.length) {\n          throw new Error(\n            `Dynamic ds is invalid\\n${errors\n              .map((e) => e.toString())\n              .join('\\n')}`,\n          );\n        }\n      }\n      return dsObj;\n    } catch (e) {\n      throw new Error(`Unable to create dynamic datasource.\\n ${e.message}`);\n    }\n  }\n}\n"]}

package/dist/indexer/fetch.module.d.ts

@@ -0,0 +1,2 @@
+export declare class FetchModule {
+}

package/dist/indexer/fetch.module.js

@@ -0,0 +1,93 @@
+"use strict";
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FetchModule = void 0;
+const common_1 = require("@nestjs/common");
+const event_emitter_1 = require("@nestjs/event-emitter");
+const node_core_1 = require("@subql/node-core");
+const api_service_stellar_1 = require("../stellar/api.service.stellar");
+const blockDispatcher_1 = require("./blockDispatcher");
+const dictionary_service_1 = require("./dictionary.service");
+const ds_processor_service_1 = require("./ds-processor.service");
+const dynamic_ds_service_1 = require("./dynamic-ds.service");
+const fetch_service_1 = require("./fetch.service");
+const indexer_manager_1 = require("./indexer.manager");
+const project_service_1 = require("./project.service");
+const sandbox_service_1 = require("./sandbox.service");
+const unfinalizedBlocks_service_1 = require("./unfinalizedBlocks.service");
+let FetchModule = class FetchModule {
+};
+FetchModule = __decorate([
+    (0, common_1.Module)({
+        providers: [
+            node_core_1.StoreService,
+            node_core_1.StoreCacheService,
+            {
+                provide: node_core_1.ApiService,
+                useFactory: async (project, connectionPoolService, eventEmitter) => {
+                    const apiService = new api_service_stellar_1.StellarApiService(project, connectionPoolService, eventEmitter);
+                    await apiService.init();
+                    return apiService;
+                },
+                inject: ['ISubqueryProject', node_core_1.ConnectionPoolService, event_emitter_1.EventEmitter2],
+            },
+            indexer_manager_1.IndexerManager,
+            node_core_1.ConnectionPoolService,
+            node_core_1.ConnectionPoolStateManager,
+            {
+                provide: node_core_1.SmartBatchService,
+                useFactory: (nodeConfig) => {
+                    return new node_core_1.SmartBatchService(nodeConfig.batchSize);
+                },
+                inject: [node_core_1.NodeConfig],
+            },
+            {
+                provide: 'IBlockDispatcher',
+                useFactory: (nodeConfig, eventEmitter, projectService, apiService, indexerManager, smartBatchService, storeService, storeCacheService, poiService, project, dynamicDsService, unfinalizedBlocks, connectionPoolState) => nodeConfig.workers !== undefined
+                    ? new blockDispatcher_1.WorkerBlockDispatcherService(nodeConfig, eventEmitter, projectService, smartBatchService, storeService, storeCacheService, poiService, project, dynamicDsService, unfinalizedBlocks, connectionPoolState)
+                    : new blockDispatcher_1.BlockDispatcherService(apiService, nodeConfig, indexerManager, eventEmitter, projectService, smartBatchService, storeService, storeCacheService, poiService, project, dynamicDsService),
+                inject: [
+                    node_core_1.NodeConfig,
+                    event_emitter_1.EventEmitter2,
+                    'IProjectService',
+                    node_core_1.ApiService,
+                    indexer_manager_1.IndexerManager,
+                    node_core_1.SmartBatchService,
+                    node_core_1.StoreService,
+                    node_core_1.StoreCacheService,
+                    node_core_1.PoiService,
+                    'ISubqueryProject',
+                    dynamic_ds_service_1.DynamicDsService,
+                    unfinalizedBlocks_service_1.UnfinalizedBlocksService,
+                    node_core_1.ConnectionPoolStateManager,
+                ],
+            },
+            fetch_service_1.FetchService,
+            node_core_1.IndexingBenchmarkService,
+            node_core_1.PoiBenchmarkService,
+            dictionary_service_1.DictionaryService,
+            sandbox_service_1.SandboxService,
+            ds_processor_service_1.DsProcessorService,
+            dynamic_ds_service_1.DynamicDsService,
+            node_core_1.PoiService,
+            node_core_1.MmrService,
+            node_core_1.MmrQueryService,
+            node_core_1.PgMmrCacheService,
+            {
+                useClass: project_service_1.ProjectService,
+                provide: 'IProjectService',
+            },
+            unfinalizedBlocks_service_1.UnfinalizedBlocksService,
+        ],
+        exports: [node_core_1.StoreService, node_core_1.MmrService, node_core_1.StoreCacheService, node_core_1.MmrQueryService],
+    })
+], FetchModule);
+exports.FetchModule = FetchModule;
+//# sourceMappingURL=fetch.module.js.map

package/dist/indexer/fetch.module.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.module.js","sourceRoot":"","sources":["../../src/indexer/fetch.module.ts"],"names":[],"mappings":";AAAA,8DAA8D;AAC9D,mCAAmC;;;;;;;;;AAEnC,2CAAwC;AACxC,yDAAsD;AACtD,gDAc0B;AAG1B,wEAAmE;AACnE,uDAG2B;AAC3B,6DAAyD;AACzD,iEAA4D;AAC5D,6DAAwD;AACxD,mDAA+C;AAC/C,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AAgHhE,IAAM,WAAW,GAAjB,MAAM,WAAW;CAAG,CAAA;AAAd,WAAW;IA9GvB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,wBAAY;YACZ,6BAAiB;YACjB;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EACf,OAAwB,EACxB,qBAAkE,EAClE,YAA2B,EAC3B,EAAE;oBACF,MAAM,UAAU,GAAG,IAAI,uCAAiB,CACtC,OAAO,EACP,qBAAqB,EACrB,YAAY,CACb,CAAC;oBACF,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,iCAAqB,EAAE,6BAAa,CAAC;aACnE;YACD,gCAAc;YACd,iCAAqB;YACrB,sCAA0B;YAC1B;gBACE,OAAO,EAAE,6BAAiB;gBAC1B,UAAU,EAAE,CAAC,UAAsB,EAAE,EAAE;oBACrC,OAAO,IAAI,6BAAiB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;gBACrD,CAAC;gBACD,MAAM,EAAE,CAAC,sBAAU,CAAC;aACrB;YACD;gBACE,OAAO,EAAE,kBAAkB;gBAC3B,UAAU,EAAE,CACV,UAAsB,EACtB,YAA2B,EAC3B,cAA8B,EAC9B,UAA6B,EAC7B,cAA8B,EAC9B,iBAAoC,EACpC,YAA0B,EAC1B,iBAAoC,EACpC,UAAsB,EACtB,OAAwB,EACxB,gBAAkC,EAClC,iBAA2C,EAC3C,mBAAqE,EACrE,EAAE,CACF,UAAU,CAAC,OAAO,KAAK,SAAS;oBAC9B,CAAC,CAAC,IAAI,8CAA4B,CAC9B,UAAU,EACV,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,OAAO,EACP,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,CACpB;oBACH,CAAC,CAAC,IAAI,wCAAsB,CACxB,UAAU,EACV,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,OAAO,EACP,gBAAgB,CACjB;gBACP,MAAM,EAAE;oBACN,sBAAU;oBACV,6BAAa;oBACb,iBAAiB;oBACjB,sBAAU;oBACV,gCAAc;oBACd,6BAAiB;oBACjB,wBAAY;oBACZ,6BAAiB;oBACjB,sBAAU;oBACV,kBAAkB;oBAClB,qCAAgB;oBAChB,oDAAwB;oBACxB,sCAA0B;iBAC3B;aACF;YACD,4BAAY;YACZ,oCAAwB;YACxB,+BAAmB;YACnB,sCAAiB;YACjB,gCAAc;YACd,yCAAkB;YAClB,qCAAgB;YAChB,sBAAU;YACV,sBAAU;YACV,2BAAe;YACf,6BAAiB;YACjB;gBACE,QAAQ,EAAE,gCAAc;gBACxB,OAAO,EAAE,iBAAiB;aAC3B;YACD,oDAAwB;SACzB;QACD,OAAO,EAAE,CAAC,wBAAY,EAAE,sBAAU,EAAE,6BAAiB,EAAE,2BAAe,CAAC;KACxE,CAAC;GACW,WAAW,CAAG;AAAd,kCAAW","sourcesContent":["// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors\n// SPDX-License-Identifier: GPL-3.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  PoiBenchmarkService,\n  IndexingBenchmarkService,\n  MmrService,\n  StoreService,\n  PoiService,\n  ApiService,\n  NodeConfig,\n  ConnectionPoolService,\n  SmartBatchService,\n  StoreCacheService,\n  PgMmrCacheService,\n  MmrQueryService,\n  ConnectionPoolStateManager,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { StellarApiConnection } from '../stellar/api.connection';\nimport { StellarApiService } from '../stellar/api.service.stellar';\nimport {\n  BlockDispatcherService,\n  WorkerBlockDispatcherService,\n} from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { FetchService } from './fetch.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n@Module({\n  providers: [\n    StoreService,\n    StoreCacheService,\n    {\n      provide: ApiService,\n      useFactory: async (\n        project: SubqueryProject,\n        connectionPoolService: ConnectionPoolService<StellarApiConnection>,\n        eventEmitter: EventEmitter2,\n      ) => {\n        const apiService = new StellarApiService(\n          project,\n          connectionPoolService,\n          eventEmitter,\n        );\n        await apiService.init();\n        return apiService;\n      },\n      inject: ['ISubqueryProject', ConnectionPoolService, EventEmitter2],\n    },\n    IndexerManager,\n    ConnectionPoolService,\n    ConnectionPoolStateManager,\n    {\n      provide: SmartBatchService,\n      useFactory: (nodeConfig: NodeConfig) => {\n        return new SmartBatchService(nodeConfig.batchSize);\n      },\n      inject: [NodeConfig],\n    },\n    {\n      provide: 'IBlockDispatcher',\n      useFactory: (\n        nodeConfig: NodeConfig,\n        eventEmitter: EventEmitter2,\n        projectService: ProjectService,\n        apiService: StellarApiService,\n        indexerManager: IndexerManager,\n        smartBatchService: SmartBatchService,\n        storeService: StoreService,\n        storeCacheService: StoreCacheService,\n        poiService: PoiService,\n        project: SubqueryProject,\n        dynamicDsService: DynamicDsService,\n        unfinalizedBlocks: UnfinalizedBlocksService,\n        connectionPoolState: ConnectionPoolStateManager<StellarApiConnection>,\n      ) =>\n        nodeConfig.workers !== undefined\n          ? new WorkerBlockDispatcherService(\n              nodeConfig,\n              eventEmitter,\n              projectService,\n              smartBatchService,\n              storeService,\n              storeCacheService,\n              poiService,\n              project,\n              dynamicDsService,\n              unfinalizedBlocks,\n              connectionPoolState,\n            )\n          : new BlockDispatcherService(\n              apiService,\n              nodeConfig,\n              indexerManager,\n              eventEmitter,\n              projectService,\n              smartBatchService,\n              storeService,\n              storeCacheService,\n              poiService,\n              project,\n              dynamicDsService,\n            ),\n      inject: [\n        NodeConfig,\n        EventEmitter2,\n        'IProjectService',\n        ApiService,\n        IndexerManager,\n        SmartBatchService,\n        StoreService,\n        StoreCacheService,\n        PoiService,\n        'ISubqueryProject',\n        DynamicDsService,\n        UnfinalizedBlocksService,\n        ConnectionPoolStateManager,\n      ],\n    },\n    FetchService,\n    IndexingBenchmarkService,\n    PoiBenchmarkService,\n    DictionaryService,\n    SandboxService,\n    DsProcessorService,\n    DynamicDsService,\n    PoiService,\n    MmrService,\n    MmrQueryService,\n    PgMmrCacheService,\n    {\n      useClass: ProjectService,\n      provide: 'IProjectService',\n    },\n    UnfinalizedBlocksService,\n  ],\n  exports: [StoreService, MmrService, StoreCacheService, MmrQueryService],\n})\nexport class FetchModule {}\n"]}

package/dist/indexer/fetch.service.d.ts

@@ -0,0 +1,33 @@
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { SchedulerRegistry } from '@nestjs/schedule';
+import { SubqlStellarProcessorOptions } from '@subql/common-stellar';
+import { NodeConfig, BaseFetchService, ApiService } from '@subql/node-core';
+import { DictionaryQueryEntry } from '@subql/types';
+import { SubqlDatasource } from '@subql/types-stellar';
+import { MetaData } from '@subql/utils';
+import { SubqueryProject } from '../configure/SubqueryProject';
+import { StellarApi } from '../stellar';
+import { IStellarBlockDispatcher } from './blockDispatcher';
+import { DictionaryService } from './dictionary.service';
+import { DsProcessorService } from './ds-processor.service';
+import { DynamicDsService } from './dynamic-ds.service';
+import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
+type GroupedSubqlProjectDs = SubqlDatasource & {
+    groupedOptions?: SubqlStellarProcessorOptions[];
+};
+export declare function buildDictionaryQueryEntries(dataSources: GroupedSubqlProjectDs[], startBlock: number): DictionaryQueryEntry[];
+export declare class FetchService extends BaseFetchService<ApiService, SubqlDatasource, IStellarBlockDispatcher, DictionaryService> {
+    private unfinalizedBlocksService;
+    constructor(apiService: ApiService, nodeConfig: NodeConfig, project: SubqueryProject, blockDispatcher: IStellarBlockDispatcher, dictionaryService: DictionaryService, dsProcessorService: DsProcessorService, dynamicDsService: DynamicDsService, unfinalizedBlocksService: UnfinalizedBlocksService, eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry);
+    get api(): StellarApi;
+    buildDictionaryQueryEntries(startBlock: number): DictionaryQueryEntry[];
+    protected getFinalizedHeight(): Promise<number>;
+    protected getBestHeight(): Promise<number>;
+    protected getChainInterval(): Promise<number>;
+    protected getChainId(): Promise<string>;
+    protected initBlockDispatcher(): Promise<void>;
+    protected validatateDictionaryMeta(metaData: MetaData): Promise<boolean>;
+    protected preLoopHook(): Promise<void>;
+    protected getModulos(): number[];
+}
+export {};

package/dist/indexer/fetch.service.js

@@ -0,0 +1,201 @@
+"use strict";
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FetchService = exports.buildDictionaryQueryEntries = void 0;
+const common_1 = require("@nestjs/common");
+const event_emitter_1 = require("@nestjs/event-emitter");
+const schedule_1 = require("@nestjs/schedule");
+const common_stellar_1 = require("@subql/common-stellar");
+const node_core_1 = require("@subql/node-core");
+const lodash_1 = require("lodash");
+const SubqueryProject_1 = require("../configure/SubqueryProject");
+const utils_stellar_1 = require("../stellar/utils.stellar");
+const yargs_1 = require("../yargs");
+const dictionary_service_1 = require("./dictionary.service");
+const ds_processor_service_1 = require("./ds-processor.service");
+const dynamic_ds_service_1 = require("./dynamic-ds.service");
+const unfinalizedBlocks_service_1 = require("./unfinalizedBlocks.service");
+const logger = (0, node_core_1.getLogger)('fetch.service');
+const BLOCK_TIME_VARIANCE = 5000;
+const INTERVAL_PERCENT = 0.9;
+function eventFilterToQueryEntry(filter, dsOptions) {
+    const queryAddressLimit = yargs_1.yargsOptions.argv['query-address-limit'];
+    const conditions = [];
+    if (Array.isArray(dsOptions)) {
+        const addresses = dsOptions.map((option) => option.address).filter(Boolean);
+        if (addresses.length > queryAddressLimit) {
+            logger.warn(`Addresses length: ${addresses.length} is exceeding limit: ${queryAddressLimit}. Consider increasing this value with the flag --query-address-limit  `);
+        }
+        if (addresses.length !== 0 && addresses.length <= queryAddressLimit) {
+            conditions.push({
+                field: 'address',
+                value: addresses,
+                matcher: 'in',
+            });
+        }
+    }
+    else {
+        if (dsOptions === null || dsOptions === void 0 ? void 0 : dsOptions.address) {
+            conditions.push({
+                field: 'address',
+                value: dsOptions.address.toLowerCase(),
+                matcher: 'equalTo',
+            });
+        }
+    }
+    if (filter.topics) {
+        for (let i = 0; i < Math.min(filter.topics.length, 4); i++) {
+            const topic = filter.topics[i];
+            if (!topic) {
+                continue;
+            }
+            const field = `topics${i}`;
+            conditions.push({
+                field,
+                value: topic,
+                matcher: 'equalTo',
+            });
+        }
+    }
+    return {
+        entity: 'events',
+        conditions,
+    };
+}
+function buildDictionaryQueryEntries(dataSources, startBlock) {
+    var _a;
+    const queryEntries = [];
+    // Only run the ds that is equal or less than startBlock
+    // sort array from lowest ds.startBlock to highest
+    const filteredDs = dataSources
+        .filter((ds) => ds.startBlock <= startBlock)
+        .sort((a, b) => a.startBlock - b.startBlock);
+    for (const ds of filteredDs) {
+        for (const handler of ds.mapping.handlers) {
+            // No filters, cant use dictionary
+            if (!handler.filter)
+                return [];
+            switch (handler.kind) {
+                case common_stellar_1.StellarHandlerKind.Event: {
+                    const filter = handler.filter;
+                    if (ds.groupedOptions) {
+                        queryEntries.push(eventFilterToQueryEntry(filter, ds.groupedOptions));
+                    }
+                    else if (((_a = ds.options) === null || _a === void 0 ? void 0 : _a.address) || filter.topics) {
+                        queryEntries.push(eventFilterToQueryEntry(filter, ds.options));
+                    }
+                    else {
+                        return [];
+                    }
+                    break;
+                }
+                default:
+            }
+        }
+    }
+    return (0, lodash_1.uniqBy)(queryEntries, (item) => `${item.entity}|${JSON.stringify((0, lodash_1.sortBy)(item.conditions, (c) => c.field))}`);
+}
+exports.buildDictionaryQueryEntries = buildDictionaryQueryEntries;
+let FetchService = class FetchService extends node_core_1.BaseFetchService {
+    constructor(apiService, nodeConfig, project, blockDispatcher, dictionaryService, dsProcessorService, dynamicDsService, unfinalizedBlocksService, eventEmitter, schedulerRegistry) {
+        super(apiService, nodeConfig, project, blockDispatcher, dictionaryService, dsProcessorService, dynamicDsService, eventEmitter, schedulerRegistry);
+        this.unfinalizedBlocksService = unfinalizedBlocksService;
+    }
+    get api() {
+        return this.apiService.unsafeApi;
+    }
+    buildDictionaryQueryEntries(startBlock) {
+        const groupdDynamicDs = Object.values((0, lodash_1.groupBy)(this.templateDynamicDatasouces, (ds) => ds.name)).map((grouped) => {
+            const options = grouped.map((ds) => ds.options);
+            const ref = grouped[0];
+            return Object.assign(Object.assign({}, ref), { groupedOptions: options });
+        });
+        // Only run the ds that is equal or less than startBlock
+        // sort array from lowest ds.startBlock to highest
+        const filteredDs = this.project.dataSources.concat(groupdDynamicDs);
+        return buildDictionaryQueryEntries(filteredDs, startBlock);
+    }
+    async getFinalizedHeight() {
+        const sequence = await this.api.getFinalizedBlockHeight();
+        const header = (0, unfinalizedBlocks_service_1.blockToHeader)(sequence);
+        this.unfinalizedBlocksService.registerFinalizedBlock(header);
+        return header.blockHeight;
+    }
+    async getBestHeight() {
+        return this.api.getBestBlockHeight();
+    }
+    // eslint-disable-next-line @typescript-eslint/require-await
+    async getChainInterval() {
+        const CHAIN_INTERVAL = (0, utils_stellar_1.calcInterval)(this.api) * INTERVAL_PERCENT;
+        return Math.min(BLOCK_TIME_VARIANCE, CHAIN_INTERVAL);
+    }
+    async getChainId() {
+        return Promise.resolve(this.api.getChainId().toString());
+    }
+    /*
+    protected getModulos(): number[] {
+      const modulos: number[] = [];
+      for (const ds of this.project.dataSources) {
+        if (isCustomDs(ds)) {
+          continue;
+        }
+        for (const handler of ds.mapping.handlers) {
+          if (
+            handler.kind === StellarHandlerKind.Block &&
+            handler.filter &&
+            handler.filter.modulo
+          ) {
+            modulos.push(handler.filter.modulo);
+          }
+        }
+      }
+      return modulos;
+    }
+    */
+    async initBlockDispatcher() {
+        await this.blockDispatcher.init(this.resetForNewDs.bind(this));
+    }
+    async validatateDictionaryMeta(metaData) {
+        return Promise.resolve(
+        // When alias is not used
+        metaData.genesisHash !== this.api.getGenesisHash() &&
+            // Case when an alias is used
+            metaData.genesisHash !== this.dictionaryService.chainId);
+    }
+    async preLoopHook() {
+        // Stellar doesn't need to do anything here
+        return Promise.resolve();
+    }
+    getModulos() {
+        //block handler not implemented yet
+        return [];
+    }
+};
+FetchService = __decorate([
+    (0, common_1.Injectable)(),
+    __param(2, (0, common_1.Inject)('ISubqueryProject')),
+    __param(3, (0, common_1.Inject)('IBlockDispatcher')),
+    __metadata("design:paramtypes", [node_core_1.ApiService,
+        node_core_1.NodeConfig,
+        SubqueryProject_1.SubqueryProject, Object, dictionary_service_1.DictionaryService,
+        ds_processor_service_1.DsProcessorService,
+        dynamic_ds_service_1.DynamicDsService,
+        unfinalizedBlocks_service_1.UnfinalizedBlocksService,
+        event_emitter_1.EventEmitter2,
+        schedule_1.SchedulerRegistry])
+], FetchService);
+exports.FetchService = FetchService;
+//# sourceMappingURL=fetch.service.js.map

package/dist/indexer/fetch.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.service.js","sourceRoot":"","sources":["../../src/indexer/fetch.service.ts"],"names":[],"mappings":";AAAA,8DAA8D;AAC9D,mCAAmC;;;;;;;;;;;;;;;AAEnC,2CAAoD;AACpD,yDAAsD;AACtD,+CAAqD;AAErD,0DAI+B;AAC/B,gDAK0B;AAI1B,mCAAiD;AACjD,kEAA+E;AAE/E,4DAAwD;AACxD,oCAAwC;AAExC,6DAAyD;AACzD,iEAA4D;AAC5D,6DAAwD;AACxD,2EAGqC;AAErC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,eAAe,CAAC,CAAC;AAE1C,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,SAAS,uBAAuB,CAC9B,MAA0B,EAC1B,SAAwE;IAExE,MAAM,iBAAiB,GAAG,oBAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAEnE,MAAM,UAAU,GAA+B,EAAE,CAAC;IAElD,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;QAC5B,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE5E,IAAI,SAAS,CAAC,MAAM,GAAG,iBAAiB,EAAE;YACxC,MAAM,CAAC,IAAI,CACT,qBAAqB,SAAS,CAAC,MAAM,wBAAwB,iBAAiB,wEAAwE,CACvJ,CAAC;SACH;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,iBAAiB,EAAE;YACnE,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;SACJ;KACF;SAAM;QACL,IAAI,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,OAAO,EAAE;YACtB,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE;gBACtC,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;SACJ;KACF;IACD,IAAI,MAAM,CAAC,MAAM,EAAE;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,EAAE;gBACV,SAAS;aACV;YACD,MAAM,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK;gBACL,KAAK,EAAE,KAAK;gBACZ,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;SACJ;KACF;IACD,OAAO;QACL,MAAM,EAAE,QAAQ;QAChB,UAAU;KACX,CAAC;AACJ,CAAC;AAKD,SAAgB,2BAA2B,CACzC,WAAoC,EACpC,UAAkB;;IAElB,MAAM,YAAY,GAA2B,EAAE,CAAC;IAEhD,wDAAwD;IACxD,kDAAkD;IAClD,MAAM,UAAU,GAAG,WAAW;SAC3B,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,UAAU,CAAC;SAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAE/C,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE;QAC3B,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;YACzC,kCAAkC;YAClC,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,OAAO,EAAE,CAAC;YAE/B,QAAQ,OAAO,CAAC,IAAI,EAAE;gBACpB,KAAK,mCAAkB,CAAC,KAAK,CAAC,CAAC;oBAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,MAA4B,CAAC;oBACpD,IAAI,EAAE,CAAC,cAAc,EAAE;wBACrB,YAAY,CAAC,IAAI,CACf,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC,CACnD,CAAC;qBACH;yBAAM,IAAI,CAAA,MAAA,EAAE,CAAC,OAAO,0CAAE,OAAO,KAAI,MAAM,CAAC,MAAM,EAAE;wBAC/C,YAAY,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;qBAChE;yBAAM;wBACL,OAAO,EAAE,CAAC;qBACX;oBACD,MAAM;iBACP;gBACD,QAAQ;aACT;SACF;KACF;IAED,OAAO,IAAA,eAAM,EACX,YAAY,EACZ,CAAC,IAAI,EAAE,EAAE,CACP,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAC9B,IAAA,eAAM,EAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACxC,EAAE,CACN,CAAC;AACJ,CAAC;AA3CD,kEA2CC;AAGM,IAAM,YAAY,GAAlB,MAAM,YAAa,SAAQ,4BAKjC;IACC,YACE,UAAsB,EACtB,UAAsB,EACM,OAAwB,EAEpD,eAAwC,EACxC,iBAAoC,EACpC,kBAAsC,EACtC,gBAAkC,EAC1B,wBAAkD,EAC1D,YAA2B,EAC3B,iBAAoC;QAEpC,KAAK,CACH,UAAU,EACV,UAAU,EACV,OAAO,EACP,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAClB,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,CAClB,CAAC;QAdM,6BAAwB,GAAxB,wBAAwB,CAA0B;IAe5D,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;IACnC,CAAC;IAED,2BAA2B,CAAC,UAAkB;QAC5C,MAAM,eAAe,GAA4B,MAAM,CAAC,MAAM,CAC5D,IAAA,gBAAO,EAAC,IAAI,CAAC,yBAAyB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CACzD,CAAC,GAAG,CAAC,CAAC,OAAyB,EAAE,EAAE;YAClC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YAChD,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvB,uCACK,GAAG,KACN,cAAc,EAAE,OAAO,IACvB;QACJ,CAAC,CAAC,CAAC;QAEH,wDAAwD;QACxD,kDAAkD;QAClD,MAAM,UAAU,GACd,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAEnD,OAAO,2BAA2B,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC7D,CAAC;IAES,KAAK,CAAC,kBAAkB;QAChC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC;QAE1D,MAAM,MAAM,GAAG,IAAA,yCAAa,EAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;QAC7D,OAAO,MAAM,CAAC,WAAW,CAAC;IAC5B,CAAC;IAES,KAAK,CAAC,aAAa;QAC3B,OAAO,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;IACvC,CAAC;IAED,4DAA4D;IAClD,KAAK,CAAC,gBAAgB;QAC9B,MAAM,cAAc,GAAG,IAAA,4BAAY,EAAC,IAAI,CAAC,GAAG,CAAC,GAAG,gBAAgB,CAAC;QAEjE,OAAO,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;IACvD,CAAC;IAES,KAAK,CAAC,UAAU;QACxB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;;;;;;;;;;;;;;MAmBE;IAEQ,KAAK,CAAC,mBAAmB;QACjC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,CAAC;IAES,KAAK,CAAC,wBAAwB,CACtC,QAAkB;QAElB,OAAO,OAAO,CAAC,OAAO;QACpB,yBAAyB;QACzB,QAAQ,CAAC,WAAW,KAAK,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE;YAChD,6BAA6B;YAC7B,QAAQ,CAAC,WAAW,KAAK,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAC1D,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,WAAW;QACzB,2CAA2C;QAC3C,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAES,UAAU;QAClB,mCAAmC;QACnC,OAAO,EAAE,CAAC;IACZ,CAAC;CACF,CAAA;AA9HY,YAAY;IADxB,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;IAC1B,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAHf,sBAAU;QACV,sBAAU;QACe,iCAAe,UAGjC,sCAAiB;QAChB,yCAAkB;QACpB,qCAAgB;QACA,oDAAwB;QAC5C,6BAAa;QACR,4BAAiB;GAjB3B,YAAY,CA8HxB;AA9HY,oCAAY","sourcesContent":["// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors\n// SPDX-License-Identifier: GPL-3.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { SchedulerRegistry } from '@nestjs/schedule';\n\nimport {\n  StellarHandlerKind,\n  StellarEventFilter,\n  SubqlStellarProcessorOptions,\n} from '@subql/common-stellar';\nimport {\n  NodeConfig,\n  BaseFetchService,\n  ApiService,\n  getLogger,\n} from '@subql/node-core';\nimport { DictionaryQueryCondition, DictionaryQueryEntry } from '@subql/types';\nimport { StellarBlock, SubqlDatasource } from '@subql/types-stellar';\nimport { MetaData } from '@subql/utils';\nimport { groupBy, sortBy, uniqBy } from 'lodash';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { StellarApi } from '../stellar';\nimport { calcInterval } from '../stellar/utils.stellar';\nimport { yargsOptions } from '../yargs';\nimport { IStellarBlockDispatcher } from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport {\n  blockToHeader,\n  UnfinalizedBlocksService,\n} from './unfinalizedBlocks.service';\n\nconst logger = getLogger('fetch.service');\n\nconst BLOCK_TIME_VARIANCE = 5000;\n\nconst INTERVAL_PERCENT = 0.9;\n\nfunction eventFilterToQueryEntry(\n  filter: StellarEventFilter,\n  dsOptions: SubqlStellarProcessorOptions | SubqlStellarProcessorOptions[],\n): DictionaryQueryEntry {\n  const queryAddressLimit = yargsOptions.argv['query-address-limit'];\n\n  const conditions: DictionaryQueryCondition[] = [];\n\n  if (Array.isArray(dsOptions)) {\n    const addresses = dsOptions.map((option) => option.address).filter(Boolean);\n\n    if (addresses.length > queryAddressLimit) {\n      logger.warn(\n        `Addresses length: ${addresses.length} is exceeding limit: ${queryAddressLimit}. Consider increasing this value with the flag --query-address-limit  `,\n      );\n    }\n\n    if (addresses.length !== 0 && addresses.length <= queryAddressLimit) {\n      conditions.push({\n        field: 'address',\n        value: addresses,\n        matcher: 'in',\n      });\n    }\n  } else {\n    if (dsOptions?.address) {\n      conditions.push({\n        field: 'address',\n        value: dsOptions.address.toLowerCase(),\n        matcher: 'equalTo',\n      });\n    }\n  }\n  if (filter.topics) {\n    for (let i = 0; i < Math.min(filter.topics.length, 4); i++) {\n      const topic = filter.topics[i];\n      if (!topic) {\n        continue;\n      }\n      const field = `topics${i}`;\n      conditions.push({\n        field,\n        value: topic,\n        matcher: 'equalTo',\n      });\n    }\n  }\n  return {\n    entity: 'events',\n    conditions,\n  };\n}\n\ntype GroupedSubqlProjectDs = SubqlDatasource & {\n  groupedOptions?: SubqlStellarProcessorOptions[];\n};\nexport function buildDictionaryQueryEntries(\n  dataSources: GroupedSubqlProjectDs[],\n  startBlock: number,\n): DictionaryQueryEntry[] {\n  const queryEntries: DictionaryQueryEntry[] = [];\n\n  // Only run the ds that is equal or less than startBlock\n  // sort array from lowest ds.startBlock to highest\n  const filteredDs = dataSources\n    .filter((ds) => ds.startBlock <= startBlock)\n    .sort((a, b) => a.startBlock - b.startBlock);\n\n  for (const ds of filteredDs) {\n    for (const handler of ds.mapping.handlers) {\n      // No filters, cant use dictionary\n      if (!handler.filter) return [];\n\n      switch (handler.kind) {\n        case StellarHandlerKind.Event: {\n          const filter = handler.filter as StellarEventFilter;\n          if (ds.groupedOptions) {\n            queryEntries.push(\n              eventFilterToQueryEntry(filter, ds.groupedOptions),\n            );\n          } else if (ds.options?.address || filter.topics) {\n            queryEntries.push(eventFilterToQueryEntry(filter, ds.options));\n          } else {\n            return [];\n          }\n          break;\n        }\n        default:\n      }\n    }\n  }\n\n  return uniqBy(\n    queryEntries,\n    (item) =>\n      `${item.entity}|${JSON.stringify(\n        sortBy(item.conditions, (c) => c.field),\n      )}`,\n  );\n}\n\n@Injectable()\nexport class FetchService extends BaseFetchService<\n  ApiService,\n  SubqlDatasource,\n  IStellarBlockDispatcher,\n  DictionaryService\n> {\n  constructor(\n    apiService: ApiService,\n    nodeConfig: NodeConfig,\n    @Inject('ISubqueryProject') project: SubqueryProject,\n    @Inject('IBlockDispatcher')\n    blockDispatcher: IStellarBlockDispatcher,\n    dictionaryService: DictionaryService,\n    dsProcessorService: DsProcessorService,\n    dynamicDsService: DynamicDsService,\n    private unfinalizedBlocksService: UnfinalizedBlocksService,\n    eventEmitter: EventEmitter2,\n    schedulerRegistry: SchedulerRegistry,\n  ) {\n    super(\n      apiService,\n      nodeConfig,\n      project,\n      blockDispatcher,\n      dictionaryService,\n      dsProcessorService,\n      dynamicDsService,\n      eventEmitter,\n      schedulerRegistry,\n    );\n  }\n\n  get api(): StellarApi {\n    return this.apiService.unsafeApi;\n  }\n\n  buildDictionaryQueryEntries(startBlock: number): DictionaryQueryEntry[] {\n    const groupdDynamicDs: GroupedSubqlProjectDs[] = Object.values(\n      groupBy(this.templateDynamicDatasouces, (ds) => ds.name),\n    ).map((grouped: SubqlProjectDs[]) => {\n      const options = grouped.map((ds) => ds.options);\n      const ref = grouped[0];\n\n      return {\n        ...ref,\n        groupedOptions: options,\n      };\n    });\n\n    // Only run the ds that is equal or less than startBlock\n    // sort array from lowest ds.startBlock to highest\n    const filteredDs: GroupedSubqlProjectDs[] =\n      this.project.dataSources.concat(groupdDynamicDs);\n\n    return buildDictionaryQueryEntries(filteredDs, startBlock);\n  }\n\n  protected async getFinalizedHeight(): Promise<number> {\n    const sequence = await this.api.getFinalizedBlockHeight();\n\n    const header = blockToHeader(sequence);\n\n    this.unfinalizedBlocksService.registerFinalizedBlock(header);\n    return header.blockHeight;\n  }\n\n  protected async getBestHeight(): Promise<number> {\n    return this.api.getBestBlockHeight();\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  protected async getChainInterval(): Promise<number> {\n    const CHAIN_INTERVAL = calcInterval(this.api) * INTERVAL_PERCENT;\n\n    return Math.min(BLOCK_TIME_VARIANCE, CHAIN_INTERVAL);\n  }\n\n  protected async getChainId(): Promise<string> {\n    return Promise.resolve(this.api.getChainId().toString());\n  }\n\n  /*\n  protected getModulos(): number[] {\n    const modulos: number[] = [];\n    for (const ds of this.project.dataSources) {\n      if (isCustomDs(ds)) {\n        continue;\n      }\n      for (const handler of ds.mapping.handlers) {\n        if (\n          handler.kind === StellarHandlerKind.Block &&\n          handler.filter &&\n          handler.filter.modulo\n        ) {\n          modulos.push(handler.filter.modulo);\n        }\n      }\n    }\n    return modulos;\n  }\n  */\n\n  protected async initBlockDispatcher(): Promise<void> {\n    await this.blockDispatcher.init(this.resetForNewDs.bind(this));\n  }\n\n  protected async validatateDictionaryMeta(\n    metaData: MetaData,\n  ): Promise<boolean> {\n    return Promise.resolve(\n      // When alias is not used\n      metaData.genesisHash !== this.api.getGenesisHash() &&\n        // Case when an alias is used\n        metaData.genesisHash !== this.dictionaryService.chainId,\n    );\n  }\n\n  protected async preLoopHook(): Promise<void> {\n    // Stellar doesn't need to do anything here\n    return Promise.resolve();\n  }\n\n  protected getModulos(): number[] {\n    //block handler not implemented yet\n    return [];\n  }\n}\n"]}

package/dist/indexer/fetch.service.spec.d.ts

@@ -0,0 +1 @@
+export {};