@subql/node-ethereum 0.4.1-13 → 0.4.1-14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -118,7 +118,8 @@ let DynamicDsService = class DynamicDsService {
|
|
|
118
118
|
});
|
|
119
119
|
}
|
|
120
120
|
async getDatasource(params) {
|
|
121
|
-
|
|
121
|
+
var _a;
|
|
122
|
+
const template = (0, lodash_1.cloneDeep)((_a = this.project.templates) === null || _a === void 0 ? void 0 : _a.find((t) => t.name === params.templateName));
|
|
122
123
|
if (!template) {
|
|
123
124
|
throw new Error(`Unable to find matching template in project for name: "${params.templateName}"`);
|
|
124
125
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dynamic-ds.service.js","sourceRoot":"","sources":["../../src/indexer/dynamic-ds.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAE5B,2CAAoD;AACpD,4DAIgC;AAChC,gDAA2D;AAC3D,yDAAiD;AACjD,qDAA+C;AAC/C,mCAAuD;AAEvD,kEAA+E;AAC/E,iEAA4D;AAE5D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEvC,MAAM,YAAY,GAAG,oBAAoB,CAAC;AAC1C,MAAM,cAAc,GAAG,KAAK,CAAC;AAStB,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAI3B,YACmB,kBAAsC,EACV,OAAwB;QADpD,uBAAkB,GAAlB,kBAAkB,CAAoB;QACV,YAAO,GAAP,OAAO,CAAiB;IACpE,CAAC;IAEJ,IAAI,CAAC,YAA0B;QAC7B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAID,KAAK,CAAC,sBAAsB,CAC1B,YAAoB,EACpB,EAAe;QAEf,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;YAC1B,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CACjC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,YAAY,CACtC,CAAC;YACF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAC5B,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EACvC,EAAE,WAAW,EAAE,EAAE,EAAE,CACpB,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB,CAC3B,MAAwB,EACxB,EAAe;QAEf,IAAI;YACF,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAE5C,MAAM,IAAI,CAAC,2BAA2B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAEnD,MAAM,CAAC,IAAI,CACT,kDAAkD,MAAM,CAAC,YAAY,GAAG,CACzE,CAAC;YAEF,IAAI,CAAC,IAAI,CAAC,YAAY;gBAAE,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;YAC/C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE3B,OAAO,EAAE,CAAC;SACX;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,gEAAgE;QAChE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;SACzD;QAED,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,8CAA8C;IAC9C,KAAK,CAAC,wBAAwB;QAC5B,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,IAAI;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;YAEvD,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CACnD,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,UAAU,WAAW,CAAC,MAAM,sBAAsB,CAAC,CAAC;YAEhE,OAAO,WAAW,CAAC;SACpB;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAED,mBAAmB,CAAC,WAAmB;QACrC,uDAAuD;QACvD,IAAI,IAAI,CAAC,aAAa,EAAE;YACtB,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,GAAG,WAAW,CAAC,CAAC;SACzD;IACH,CAAC;IAEO,KAAK,CAAC,0BAA0B,CACtC,WAAoB;;QAEpB,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE9D,IAAI,OAAO,GAAuB,EAAE,CAAC;QAErC,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,KAAgB,mCAAI,IAAI,CAClC,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,EAAE;YACtB,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;SAC5B;QAED,IAAI,WAAW,KAAK,SAAS,EAAE;YAC7B,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAA,MAAA,IAAI,CAAC,aAAa,0CAAG,cAAc,GAAG,WAAW,CAAC,mCAAI,IAAI,CAC3D,CAAC;YACF,IAAI,WAAW,CAAC,MAAM,EAAE;gBACtB,OAAO,GAAG,IAAA,kBAAS,EAAC,OAAO,EAAE,WAAW,EAAE,gBAAO,CAAC,CAAC;aACpD;SACF;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,QAA0B,EAC1B,EAAe;QAEf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE5E,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,YAAY;aACpB,MAAM,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;aACpE,IAAI,CAAC,GAAG,EAAE;YACT,IAAI,CAAC,aAAa,mCACb,IAAI,CAAC,aAAa,KACrB,CAAC,cAAc,GAAG,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,GAClD,CAAC;QACJ,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,MAAwB;QAExB,MAAM,QAAQ,GAAG,IAAA,kBAAS,EACxB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,YAAY,CAAC,CACnE,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,KAAK,CACb,0DAA0D,MAAM,CAAC,YAAY,GAAG,CACjF,CAAC;SACH;QAED,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,UAAU,EAAE,MAAM,CAAC,UAAU,GACZ,CAAC;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC;QAClB,IAAI;YACF,IAAI,IAAA,4BAAU,EAAC,KAAK,CAAC,EAAE;gBACrB,KAAK,CAAC,SAAS,CAAC,OAAO,mCAClB,KAAK,CAAC,SAAS,CAAC,OAAO,GACvB,MAAM,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACzD;iBAAM,IAAI,IAAA,6BAAW,EAAC,KAAK,CAAC,EAAE;gBAC7B,KAAK,CAAC,OAAO,mCACR,KAAK,CAAC,OAAO,GACb,MAAM,CAAC,IAAI,CACf,CAAC;gBAEF,MAAM,QAAQ,GAAG,IAAA,gCAAY,EAC3B,qDAAmC,EACnC,KAAK,CACN,CAAC;gBAEF,MAAM,MAAM,GAAG,IAAA,8BAAY,EAAC,QAAQ,EAAE;oBACpC,SAAS,EAAE,IAAI;oBACf,oBAAoB,EAAE,KAAK;iBAC5B,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,EAAE;oBACjB,MAAM,IAAI,KAAK,CACb,0BAA0B,MAAM;yBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;yBACxB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;iBACH;aACF;YACD,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACxE;IACH,CAAC;CACF,CAAA;AA/LY,gBAAgB;IAD5B,IAAA,mBAAU,GAAE;IAOR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCADU,yCAAkB;QACD,iCAAe;GAN5D,gBAAgB,CA+L5B;AA/LY,4CAAgB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n EthereumRuntimeDataSourceV0_3_0Impl,\n isCustomDs,\n isRuntimeDs,\n} from '@subql/common-ethereum';\nimport { getLogger, MetadataRepo } from '@subql/node-core';\nimport { plainToClass } from 'class-transformer';\nimport { validateSync } from 'class-validator';\nimport { cloneDeep, isEqual, unionWith } from 'lodash';\nimport { Transaction } from 'sequelize/types';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { DsProcessorService } from './ds-processor.service';\n\nconst logger = getLogger('dynamic-ds');\n\nconst METADATA_KEY = 'dynamicDatasources';\nconst TEMP_DS_PREFIX = 'ds_';\n\ninterface DatasourceParams {\n templateName: string;\n args?: Record<string, unknown>;\n startBlock: number;\n}\n\n@Injectable()\nexport class DynamicDsService {\n private metaDataRepo: MetadataRepo;\n private tempDsRecords: Record<string, string>;\n\n constructor(\n private readonly dsProcessorService: DsProcessorService,\n @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n ) {}\n\n init(metaDataRepo: MetadataRepo): void {\n this.metaDataRepo = metaDataRepo;\n }\n\n private _datasources: SubqlProjectDs[];\n\n async resetDynamicDatasource(\n targetHeight: number,\n tx: Transaction,\n ): Promise<void> {\n const dynamicDs = await this.getDynamicDatasourceParams();\n if (dynamicDs.length !== 0) {\n const filteredDs = dynamicDs.filter(\n (ds) => ds.startBlock <= targetHeight,\n );\n const dsRecords = JSON.stringify(filteredDs);\n await this.metaDataRepo.upsert(\n { key: METADATA_KEY, value: dsRecords },\n { transaction: tx },\n );\n }\n }\n\n async createDynamicDatasource(\n params: DatasourceParams,\n tx: Transaction,\n ): Promise<SubqlProjectDs> {\n try {\n const ds = await this.getDatasource(params);\n\n await this.saveDynamicDatasourceParams(params, tx);\n\n logger.info(\n `Created new dynamic datasource from template: \"${params.templateName}\"`,\n );\n\n if (!this._datasources) this._datasources = [];\n this._datasources.push(ds);\n\n return ds;\n } catch (e) {\n logger.error(e.message);\n process.exit(1);\n }\n }\n\n async getDynamicDatasources(): Promise<SubqlProjectDs[]> {\n // Workers should not cache this result in order to keep in sync\n if (!this._datasources) {\n this._datasources = await this.loadDynamicDatasources();\n }\n\n return this._datasources;\n }\n\n // This is used to sync between worker threads\n async reloadDynamicDatasources(): Promise<void> {\n this._datasources = await this.loadDynamicDatasources();\n }\n\n private async loadDynamicDatasources(): Promise<SubqlProjectDs[]> {\n try {\n const params = await this.getDynamicDatasourceParams();\n\n const dataSources = await Promise.all(\n params.map((params) => this.getDatasource(params)),\n );\n\n logger.info(`Loaded ${dataSources.length} dynamic datasources`);\n\n return dataSources;\n } catch (e) {\n logger.error(`Unable to get dynamic datasources:\\n${e.message}`);\n process.exit(1);\n }\n }\n\n deleteTempDsRecords(blockHeight: number): void {\n // Main thread will not have tempDsRecords with workers\n if (this.tempDsRecords) {\n delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];\n }\n }\n\n private async getDynamicDatasourceParams(\n blockHeight?: number,\n ): Promise<DatasourceParams[]> {\n assert(this.metaDataRepo, `Model _metadata does not exist`);\n const record = await this.metaDataRepo.findByPk(METADATA_KEY);\n\n let results: DatasourceParams[] = [];\n\n const metaResults: DatasourceParams[] = JSON.parse(\n (record?.value as string) ?? '[]',\n );\n if (metaResults.length) {\n results = [...metaResults];\n }\n\n if (blockHeight !== undefined) {\n const tempResults: DatasourceParams[] = JSON.parse(\n this.tempDsRecords?.[TEMP_DS_PREFIX + blockHeight] ?? '[]',\n );\n if (tempResults.length) {\n results = unionWith(results, tempResults, isEqual);\n }\n }\n\n return results;\n }\n\n private async saveDynamicDatasourceParams(\n dsParams: DatasourceParams,\n tx: Transaction,\n ): Promise<void> {\n const existing = await this.getDynamicDatasourceParams(dsParams.startBlock);\n\n assert(this.metaDataRepo, `Model _metadata does not exist`);\n const dsRecords = JSON.stringify([...existing, dsParams]);\n await this.metaDataRepo\n .upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx })\n .then(() => {\n this.tempDsRecords = {\n ...this.tempDsRecords,\n [TEMP_DS_PREFIX + dsParams.startBlock]: dsRecords,\n };\n });\n }\n\n private async getDatasource(\n params: DatasourceParams,\n ): Promise<SubqlProjectDs> {\n const template = cloneDeep(\n this.project.templates.find((t) => t.name === params.templateName),\n );\n\n if (!template) {\n throw new Error(\n `Unable to find matching template in project for name: \"${params.templateName}\"`,\n );\n }\n\n const dsObj = {\n ...template,\n startBlock: params.startBlock,\n } as SubqlProjectDs;\n delete dsObj.name;\n try {\n if (isCustomDs(dsObj)) {\n dsObj.processor.options = {\n ...dsObj.processor.options,\n ...params.args,\n };\n await this.dsProcessorService.validateCustomDs([dsObj]);\n } else if (isRuntimeDs(dsObj)) {\n dsObj.options = {\n ...dsObj.options,\n ...params.args,\n };\n\n const parsedDs = plainToClass(\n EthereumRuntimeDataSourceV0_3_0Impl,\n dsObj,\n );\n\n const errors = validateSync(parsedDs, {\n whitelist: true,\n forbidNonWhitelisted: false,\n });\n if (errors.length) {\n throw new Error(\n `Dynamic ds is invalid\\n${errors\n .map((e) => e.toString())\n .join('\\n')}`,\n );\n }\n }\n return dsObj;\n } catch (e) {\n throw new Error(`Unable to create dynamic datasource.\\n ${e.message}`);\n }\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"dynamic-ds.service.js","sourceRoot":"","sources":["../../src/indexer/dynamic-ds.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAE5B,2CAAoD;AACpD,4DAIgC;AAChC,gDAA2D;AAC3D,yDAAiD;AACjD,qDAA+C;AAC/C,mCAAuD;AAEvD,kEAA+E;AAC/E,iEAA4D;AAE5D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEvC,MAAM,YAAY,GAAG,oBAAoB,CAAC;AAC1C,MAAM,cAAc,GAAG,KAAK,CAAC;AAStB,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAI3B,YACmB,kBAAsC,EACV,OAAwB;QADpD,uBAAkB,GAAlB,kBAAkB,CAAoB;QACV,YAAO,GAAP,OAAO,CAAiB;IACpE,CAAC;IAEJ,IAAI,CAAC,YAA0B;QAC7B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAID,KAAK,CAAC,sBAAsB,CAC1B,YAAoB,EACpB,EAAe;QAEf,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;YAC1B,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CACjC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,YAAY,CACtC,CAAC;YACF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAC5B,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EACvC,EAAE,WAAW,EAAE,EAAE,EAAE,CACpB,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB,CAC3B,MAAwB,EACxB,EAAe;QAEf,IAAI;YACF,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAE5C,MAAM,IAAI,CAAC,2BAA2B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAEnD,MAAM,CAAC,IAAI,CACT,kDAAkD,MAAM,CAAC,YAAY,GAAG,CACzE,CAAC;YAEF,IAAI,CAAC,IAAI,CAAC,YAAY;gBAAE,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;YAC/C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE3B,OAAO,EAAE,CAAC;SACX;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,gEAAgE;QAChE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;SACzD;QAED,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,8CAA8C;IAC9C,KAAK,CAAC,wBAAwB;QAC5B,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,IAAI;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;YAEvD,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CACnD,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,UAAU,WAAW,CAAC,MAAM,sBAAsB,CAAC,CAAC;YAEhE,OAAO,WAAW,CAAC;SACpB;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAED,mBAAmB,CAAC,WAAmB;QACrC,uDAAuD;QACvD,IAAI,IAAI,CAAC,aAAa,EAAE;YACtB,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,GAAG,WAAW,CAAC,CAAC;SACzD;IACH,CAAC;IAEO,KAAK,CAAC,0BAA0B,CACtC,WAAoB;;QAEpB,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE9D,IAAI,OAAO,GAAuB,EAAE,CAAC;QAErC,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,KAAgB,mCAAI,IAAI,CAClC,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,EAAE;YACtB,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;SAC5B;QAED,IAAI,WAAW,KAAK,SAAS,EAAE;YAC7B,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAA,MAAA,IAAI,CAAC,aAAa,0CAAG,cAAc,GAAG,WAAW,CAAC,mCAAI,IAAI,CAC3D,CAAC;YACF,IAAI,WAAW,CAAC,MAAM,EAAE;gBACtB,OAAO,GAAG,IAAA,kBAAS,EAAC,OAAO,EAAE,WAAW,EAAE,gBAAO,CAAC,CAAC;aACpD;SACF;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,QAA0B,EAC1B,EAAe;QAEf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE5E,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,YAAY;aACpB,MAAM,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;aACpE,IAAI,CAAC,GAAG,EAAE;YACT,IAAI,CAAC,aAAa,mCACb,IAAI,CAAC,aAAa,KACrB,CAAC,cAAc,GAAG,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,GAClD,CAAC;QACJ,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,MAAwB;;QAExB,MAAM,QAAQ,GAAG,IAAA,kBAAS,EACxB,MAAA,IAAI,CAAC,OAAO,CAAC,SAAS,0CAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,YAAY,CAAC,CACpE,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,KAAK,CACb,0DAA0D,MAAM,CAAC,YAAY,GAAG,CACjF,CAAC;SACH;QAED,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,UAAU,EAAE,MAAM,CAAC,UAAU,GACZ,CAAC;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC;QAClB,IAAI;YACF,IAAI,IAAA,4BAAU,EAAC,KAAK,CAAC,EAAE;gBACrB,KAAK,CAAC,SAAS,CAAC,OAAO,mCAClB,KAAK,CAAC,SAAS,CAAC,OAAO,GACvB,MAAM,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACzD;iBAAM,IAAI,IAAA,6BAAW,EAAC,KAAK,CAAC,EAAE;gBAC7B,KAAK,CAAC,OAAO,mCACR,KAAK,CAAC,OAAO,GACb,MAAM,CAAC,IAAI,CACf,CAAC;gBAEF,MAAM,QAAQ,GAAG,IAAA,gCAAY,EAC3B,qDAAmC,EACnC,KAAK,CACN,CAAC;gBAEF,MAAM,MAAM,GAAG,IAAA,8BAAY,EAAC,QAAQ,EAAE;oBACpC,SAAS,EAAE,IAAI;oBACf,oBAAoB,EAAE,KAAK;iBAC5B,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,EAAE;oBACjB,MAAM,IAAI,KAAK,CACb,0BAA0B,MAAM;yBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;yBACxB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;iBACH;aACF;YACD,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACxE;IACH,CAAC;CACF,CAAA;AA/LY,gBAAgB;IAD5B,IAAA,mBAAU,GAAE;IAOR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCADU,yCAAkB;QACD,iCAAe;GAN5D,gBAAgB,CA+L5B;AA/LY,4CAAgB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n EthereumRuntimeDataSourceV0_3_0Impl,\n isCustomDs,\n isRuntimeDs,\n} from '@subql/common-ethereum';\nimport { getLogger, MetadataRepo } from '@subql/node-core';\nimport { plainToClass } from 'class-transformer';\nimport { validateSync } from 'class-validator';\nimport { cloneDeep, isEqual, unionWith } from 'lodash';\nimport { Transaction } from 'sequelize/types';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { DsProcessorService } from './ds-processor.service';\n\nconst logger = getLogger('dynamic-ds');\n\nconst METADATA_KEY = 'dynamicDatasources';\nconst TEMP_DS_PREFIX = 'ds_';\n\ninterface DatasourceParams {\n templateName: string;\n args?: Record<string, unknown>;\n startBlock: number;\n}\n\n@Injectable()\nexport class DynamicDsService {\n private metaDataRepo: MetadataRepo;\n private tempDsRecords: Record<string, string>;\n\n constructor(\n private readonly dsProcessorService: DsProcessorService,\n @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n ) {}\n\n init(metaDataRepo: MetadataRepo): void {\n this.metaDataRepo = metaDataRepo;\n }\n\n private _datasources: SubqlProjectDs[];\n\n async resetDynamicDatasource(\n targetHeight: number,\n tx: Transaction,\n ): Promise<void> {\n const dynamicDs = await this.getDynamicDatasourceParams();\n if (dynamicDs.length !== 0) {\n const filteredDs = dynamicDs.filter(\n (ds) => ds.startBlock <= targetHeight,\n );\n const dsRecords = JSON.stringify(filteredDs);\n await this.metaDataRepo.upsert(\n { key: METADATA_KEY, value: dsRecords },\n { transaction: tx },\n );\n }\n }\n\n async createDynamicDatasource(\n params: DatasourceParams,\n tx: Transaction,\n ): Promise<SubqlProjectDs> {\n try {\n const ds = await this.getDatasource(params);\n\n await this.saveDynamicDatasourceParams(params, tx);\n\n logger.info(\n `Created new dynamic datasource from template: \"${params.templateName}\"`,\n );\n\n if (!this._datasources) this._datasources = [];\n this._datasources.push(ds);\n\n return ds;\n } catch (e) {\n logger.error(e.message);\n process.exit(1);\n }\n }\n\n async getDynamicDatasources(): Promise<SubqlProjectDs[]> {\n // Workers should not cache this result in order to keep in sync\n if (!this._datasources) {\n this._datasources = await this.loadDynamicDatasources();\n }\n\n return this._datasources;\n }\n\n // This is used to sync between worker threads\n async reloadDynamicDatasources(): Promise<void> {\n this._datasources = await this.loadDynamicDatasources();\n }\n\n private async loadDynamicDatasources(): Promise<SubqlProjectDs[]> {\n try {\n const params = await this.getDynamicDatasourceParams();\n\n const dataSources = await Promise.all(\n params.map((params) => this.getDatasource(params)),\n );\n\n logger.info(`Loaded ${dataSources.length} dynamic datasources`);\n\n return dataSources;\n } catch (e) {\n logger.error(`Unable to get dynamic datasources:\\n${e.message}`);\n process.exit(1);\n }\n }\n\n deleteTempDsRecords(blockHeight: number): void {\n // Main thread will not have tempDsRecords with workers\n if (this.tempDsRecords) {\n delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];\n }\n }\n\n private async getDynamicDatasourceParams(\n blockHeight?: number,\n ): Promise<DatasourceParams[]> {\n assert(this.metaDataRepo, `Model _metadata does not exist`);\n const record = await this.metaDataRepo.findByPk(METADATA_KEY);\n\n let results: DatasourceParams[] = [];\n\n const metaResults: DatasourceParams[] = JSON.parse(\n (record?.value as string) ?? '[]',\n );\n if (metaResults.length) {\n results = [...metaResults];\n }\n\n if (blockHeight !== undefined) {\n const tempResults: DatasourceParams[] = JSON.parse(\n this.tempDsRecords?.[TEMP_DS_PREFIX + blockHeight] ?? '[]',\n );\n if (tempResults.length) {\n results = unionWith(results, tempResults, isEqual);\n }\n }\n\n return results;\n }\n\n private async saveDynamicDatasourceParams(\n dsParams: DatasourceParams,\n tx: Transaction,\n ): Promise<void> {\n const existing = await this.getDynamicDatasourceParams(dsParams.startBlock);\n\n assert(this.metaDataRepo, `Model _metadata does not exist`);\n const dsRecords = JSON.stringify([...existing, dsParams]);\n await this.metaDataRepo\n .upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx })\n .then(() => {\n this.tempDsRecords = {\n ...this.tempDsRecords,\n [TEMP_DS_PREFIX + dsParams.startBlock]: dsRecords,\n };\n });\n }\n\n private async getDatasource(\n params: DatasourceParams,\n ): Promise<SubqlProjectDs> {\n const template = cloneDeep(\n this.project.templates?.find((t) => t.name === params.templateName),\n );\n\n if (!template) {\n throw new Error(\n `Unable to find matching template in project for name: \"${params.templateName}\"`,\n );\n }\n\n const dsObj = {\n ...template,\n startBlock: params.startBlock,\n } as SubqlProjectDs;\n delete dsObj.name;\n try {\n if (isCustomDs(dsObj)) {\n dsObj.processor.options = {\n ...dsObj.processor.options,\n ...params.args,\n };\n await this.dsProcessorService.validateCustomDs([dsObj]);\n } else if (isRuntimeDs(dsObj)) {\n dsObj.options = {\n ...dsObj.options,\n ...params.args,\n };\n\n const parsedDs = plainToClass(\n EthereumRuntimeDataSourceV0_3_0Impl,\n dsObj,\n );\n\n const errors = validateSync(parsedDs, {\n whitelist: true,\n forbidNonWhitelisted: false,\n });\n if (errors.length) {\n throw new Error(\n `Dynamic ds is invalid\\n${errors\n .map((e) => e.toString())\n .join('\\n')}`,\n );\n }\n }\n return dsObj;\n } catch (e) {\n throw new Error(`Unable to create dynamic datasource.\\n ${e.message}`);\n }\n }\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@subql/node-ethereum",
|
|
3
|
-
"version": "0.4.1-
|
|
3
|
+
"version": "0.4.1-14",
|
|
4
4
|
"description": "",
|
|
5
5
|
"author": "Ian He",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -74,5 +74,5 @@
|
|
|
74
74
|
"/dist",
|
|
75
75
|
"/bin"
|
|
76
76
|
],
|
|
77
|
-
"stableVersion": "0.4.1-
|
|
77
|
+
"stableVersion": "0.4.1-13"
|
|
78
78
|
}
|