@subql/node-ethereum 0.4.1-storeCache-0.0.6 → 1.0.1-0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +3 -1
  2. package/dist/.tsbuildinfo +1 -1
  3. package/dist/app.module.js.map +1 -1
  4. package/dist/configure/SubqueryProject.d.ts +1 -1
  5. package/dist/configure/SubqueryProject.js +3 -0
  6. package/dist/configure/SubqueryProject.js.map +1 -1
  7. package/dist/configure/configure.module.js +2 -2
  8. package/dist/configure/configure.module.js.map +1 -1
  9. package/dist/ethereum/api.connection.d.ts +11 -0
  10. package/dist/ethereum/api.connection.js +27 -0
  11. package/dist/ethereum/api.connection.js.map +1 -0
  12. package/dist/ethereum/api.ethereum.d.ts +3 -0
  13. package/dist/ethereum/api.ethereum.js +25 -5
  14. package/dist/ethereum/api.ethereum.js.map +1 -1
  15. package/dist/ethereum/api.service.ethereum.d.ts +5 -4
  16. package/dist/ethereum/api.service.ethereum.js +49 -18
  17. package/dist/ethereum/api.service.ethereum.js.map +1 -1
  18. package/dist/ethereum/api.service.ethereum.test.js +32 -20
  19. package/dist/ethereum/api.service.ethereum.test.js.map +1 -1
  20. package/dist/ethereum/ethers/web/index.d.ts +0 -1
  21. package/dist/ethereum/ethers/web/types.d.ts +0 -1
  22. package/dist/indexer/blockDispatcher/block-dispatcher.service.js.map +1 -1
  23. package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js +0 -1
  24. package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js.map +1 -1
  25. package/dist/indexer/dictionary.service.js.map +1 -1
  26. package/dist/indexer/ds-processor.service.js.map +1 -1
  27. package/dist/indexer/dynamic-ds.service.js.map +1 -1
  28. package/dist/indexer/fetch.module.js +11 -3
  29. package/dist/indexer/fetch.module.js.map +1 -1
  30. package/dist/indexer/fetch.service.d.ts +1 -1
  31. package/dist/indexer/fetch.service.js +30 -32
  32. package/dist/indexer/fetch.service.js.map +1 -1
  33. package/dist/indexer/indexer.manager.js.map +1 -1
  34. package/dist/indexer/indexer.module.js +4 -4
  35. package/dist/indexer/indexer.module.js.map +1 -1
  36. package/dist/indexer/project.service.js +1 -3
  37. package/dist/indexer/project.service.js.map +1 -1
  38. package/dist/indexer/sandbox.service.js.map +1 -1
  39. package/dist/indexer/unfinalizedBlocks.service.js +0 -1
  40. package/dist/indexer/unfinalizedBlocks.service.js.map +1 -1
  41. package/dist/indexer/worker/worker.js +1 -8
  42. package/dist/indexer/worker/worker.js.map +1 -1
  43. package/dist/indexer/worker/worker.module.js.map +1 -1
  44. package/dist/indexer/worker/worker.service.js.map +1 -1
  45. package/dist/indexer/worker/worker.unfinalizedBlocks.service.js.map +1 -1
  46. package/dist/meta/meta.controller.js.map +1 -1
  47. package/dist/meta/meta.module.js.map +1 -1
  48. package/dist/meta/meta.service.js.map +1 -1
  49. package/dist/subcommands/forceClean.module.js.map +1 -1
  50. package/dist/subcommands/forceClean.service.js.map +1 -1
  51. package/dist/subcommands/reindex.module.js.map +1 -1
  52. package/dist/subcommands/reindex.service.js +1 -2
  53. package/dist/subcommands/reindex.service.js.map +1 -1
  54. package/dist/yargs.d.ts +31 -28
  55. package/dist/yargs.js +30 -25
  56. package/dist/yargs.js.map +1 -1
  57. package/package.json +7 -7
package/dist/indexer/fetch.service.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"fetch.service.js","sourceRoot":"","sources":["../../src/indexer/fetch.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAA2E;AAC3E,yDAAsD;AACtD,+CAA+D;AAE/D,4DAMgC;AAChC,gDAU0B;AAO1B,mCAAiE;AACjE,kEAA+E;AAC/E,+DAA0D;AAC1D,4CAAkE;AAElE,6DAAyD;AACzD,6DAAwD;AACxD,2EAAuE;AAEvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,OAAO,CAAC,CAAC;AAClC,IAAI,mBAAmB,GAAG,IAAI,CAAC;AAC/B,MAAM,yBAAyB,GAAG,KAAK,CAAC;AACxC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,SAAS,uBAAuB,CAC9B,MAAyB,EACzB,SAA0E;IAE1E,MAAM,UAAU,GAA+B,EAAE,CAAC;IAElD,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;QAC5B,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE5E,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,mBAAmB,EAAE;YACrE,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,eAAe;aACzB,CAAC,CAAC;SACJ;KACF;SAAM;QACL,IAAI,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,OAAO,EAAE;YACtB,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE;gBACtC,qBAAqB;aACtB,CAAC,CAAC;SACJ;KACF;IACD,IAAI,MAAM,CAAC,MAAM,EAAE;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,EAAE;gBACV,SAAS;aACV;YACD,MAAM,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK;gBACL,KAAK,EAAE,IAAA,qBAAY,EAAC,KAAK,CAAC;gBAC1B,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;SACJ;KACF;IACD,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,MAAiC;IAEjC,MAAM,UAAU,GAA+B,EAAE,CAAC;IAClD,IAAI,MAAM,CAAC,IAAI,EAAE;QACf,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE;SACjC,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,EAAE,EAAE;QACb,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE;SAC/B,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,QAAQ,EAAE;QACnB,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,IAAA,0BAAiB,EAAC,MAAM,CAAC,QAAQ,CAAC;YACzC,OAAO,EAAE,SAAS;SACnB,CAAC,CAAC;KACJ;IACD,OAAO;QACL,MAAM,EAAE,iBAAiB;QACzB,UAAU;KACX,CAAC;AACJ,CAAC;AAGM,IAAM,YAAY,GAAlB,MAAM,YAAY;IAUvB,YACU,UAAsB,EACtB,UAAsB,EACM,OAAwB,EAEpD,eAAyC,EACzC,iBAAoC,EACpC,gBAAkC,EAClC,wBAAkD,EAClD,YAA2B,EAC3B,iBAAoC;QATpC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QAEpD,oBAAe,GAAf,eAAe,CAA0B;QACzC,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,iBAAY,GAAZ,YAAY,CAAe;QAC3B,sBAAiB,GAAjB,iBAAiB,CAAmB;QAjBtC,eAAU,GAAG,KAAK,CAAC;QAGnB,6BAAwB,GAAG,IAAI,CAAC;QAEhC,iBAAY,GAAa,EAAE,CAAC;QAclC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,qBAAqB;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,+BAA+B;QACnC,IAAI,CAAC,yBAAyB;YAC5B,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;IACxD,CAAC;IAED,2BAA2B,CAAC,UAAkB;;QAC5C,MAAM,YAAY,GAA2B,EAAE,CAAC;QAMhD,MAAM,eAAe,GAA4B,MAAM,CAAC,MAAM,CAC5D,IAAA,gBAAO,EAAC,IAAI,CAAC,yBAAyB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CACzD,CAAC,GAAG,CAAC,CAAC,OAAyB,EAAE,EAAE;YAClC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YAChD,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvB,uCACK,GAAG,KACN,cAAc,EAAE,OAAO,IACvB;QACJ,CAAC,CAAC,CAAC;QAEH,wDAAwD;QACxD,kDAAkD;QAClD,MAAM,UAAU,GAA4B,IAAI,CAAC,OAAO,CAAC,WAAW;aACjE,MAAM,CAAC,eAAe,CAAC;aACvB,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,UAAU,CAAC;aAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAE/C,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE;YAC3B,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,kCAAkC;gBAClC,IAAI,CAAC,OAAO,CAAC,MAAM;oBAAE,OAAO,EAAE,CAAC;gBAE/B,QAAQ,OAAO,CAAC,IAAI,EAAE;oBACpB,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,EAAE,CAAC;oBACZ,KAAK,qCAAmB,CAAC,IAAI,CAAC,CAAC;wBAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAmC,CAAC;wBAC3D,IACE,MAAM,CAAC,IAAI,KAAK,SAAS;4BACzB,MAAM,CAAC,EAAE,KAAK,SAAS;4BACvB,MAAM,CAAC,QAAQ,EACf;4BACA,YAAY,CAAC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC;yBACnD;6BAAM;4BACL,OAAO,EAAE,CAAC;yBACX;wBACD,MAAM;qBACP;oBACD,KAAK,qCAAmB,CAAC,KAAK,CAAC,CAAC;wBAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,MAA2B,CAAC;wBACnD,IAAI,EAAE,CAAC,cAAc,EAAE;4BACrB,YAAY,CAAC,IAAI,CACf,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC,CACnD,CAAC;yBACH;6BAAM,IAAI,CAAA,MAAA,EAAE,CAAC,OAAO,0CAAE,OAAO,KAAI,MAAM,CAAC,MAAM,EAAE;4BAC/C,YAAY,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;yBAChE;6BAAM;4BACL,OAAO,EAAE,CAAC;yBACX;wBACD,MAAM;qBACP;oBACD,QAAQ;iBACT;aACF;SACF;QAED,OAAO,IAAA,eAAM,EACX,YAAY,EACZ,CAAC,IAAI,EAAE,EAAE,CACP,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAC9B,IAAA,eAAM,EAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACxC,EAAE,CACN,CAAC;IACJ,CAAC;IAED,gBAAgB;QACd,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAC5C,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,EAC/D,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAC5C,CAAC;IACJ,CAAC;IAED,IAAY,aAAa;;QACvB,OAAO,CACL,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU;YACjC,IAAI,CAAC,wBAAwB;YAC7B,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,yBAAyB,CAChD,MAAA,IAAI,CAAC,eAAe,CAAC,oBAAoB,mCACvC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CACnE,CAAC,MAAM,CACT,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,WAAmB;;QAC5B,IAAI,CAAA,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,0CAAE,YAAY,MAAK,SAAS,EAAE;YACpD,IAAI,CAAC,YAAY,GAAG,IAAA,iCAAqB,EACvC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAClC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,WAAW,CAAC,CAAC;SACvC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE;YACZ,MAAM,cAAc,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,GAAG,CAAC,GAAG,gBAAgB,CAAC;YAEjE,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;YAEpE,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAChC,oBAAoB,EACpB,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,gBAAgB,EAAE,EAAE,mBAAmB,CAAC,CACrE,CAAC;SACH;QACD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAE7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;YACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE;YACnC,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;SAChE;QAED,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE9B,4DAA4D;QAC5D,iHAAiH;QACjH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,CAAC;QAE5D,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QAEzD,IAAI,YAAY,EAAE;YAChB,IAAI,CAAC,iBAAiB,CAAC,wBAAwB,CAC7C,QAAQ,CAAC,SAAS,CAAC,WAAW,CAC/B,CAAC;SACH;QAED,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,wBAAwB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAGD,eAAe;QACb,IAAI,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE;YACvC,MAAM,KAAK,GAAG,IAAA,4BAAgB,EAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAErE,IAAI,IAAI,CAAC,cAAc,KAAK,KAAK,EAAE;gBACjC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;aAC7B;SACF;IACH,CAAC;IAGK,AAAN,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC9D,OAAO;SACR;QACD,IAAI;YACF,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,aAAa,sBAAsB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC/D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC3D,sBAAsB,CACvB,CAAC;YACF,IAAI,IAAI,CAAC,qBAAqB,KAAK,sBAAsB,EAAE;gBACzD,IAAI,CAAC,qBAAqB,GAAG,sBAAsB,CAAC;gBACpD,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,eAAe,CAAC,CAAC;gBACtE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;oBACtC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;wBACtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;4BAC/C,MAAM,EAAE,IAAI,CAAC,qBAAqB;yBACnC,CAAC,CAAC;qBACJ;iBACF;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,2CAA2C,CAAC,CAAC;SAC7D;IACH,CAAC;IAGK,AAAN,KAAK,CAAC,gBAAgB;QACpB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,OAAO;SACR;QACD,IAAI;YACF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC9D,MAAM,CAAC,KAAK,CAAC,QAAQ,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACrD,IAAI,IAAI,CAAC,gBAAgB,KAAK,iBAAiB,EAAE;gBAC/C,IAAI,CAAC,gBAAgB,GAAG,iBAAiB,CAAC;gBAC1C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,SAAS,EAAE;oBAC7C,MAAM,EAAE,IAAI,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;gBAEH,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;oBACrC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;wBAC/C,MAAM,EAAE,IAAI,CAAC,gBAAgB;qBAC9B,CAAC,CAAC;iBACJ;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,sCAAsC,CAAC,CAAC;SACxD;IACH,CAAC;IACO,KAAK,CAAC,SAAS,CAAC,eAAuB;QAC7C,MAAM,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IAClD,CAAC;IAED,UAAU;QACR,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;YACzC,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;gBAClB,SAAS;aACV;YACD,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IACE,OAAO,CAAC,IAAI,KAAK,qCAAmB,CAAC,KAAK;oBAC1C,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,MAAM,CAAC,MAAM,EACrB;oBACA,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;iBACrC;aACF;SACF;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,eAAe,CAAC,WAAmB,EAAE,SAAiB;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,WAAW,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE;YAC5C,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE;gBACpC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACtB;SACF;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,uBAAuB,CAAC,gBAAwB;QAC9C,OAAO,IAAI,CAAC,eAAe,CACzB,gBAAgB,EAChB,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACxD,gBAAgB,CACnB,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,eAAuB;QAC/C,IAAI,gBAAwB,CAAC;QAC7B,IAAI,eAAuB,CAAC;QAC5B,MAAM,QAAQ,GAAG,EAAE,CAAC,MAAM,CACxB,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC7D,CAAC;QAEF,MAAM,mBAAmB,GAAG,GAAW,EAAE;YACvC,OAAO,IAAI,CAAC,eAAe,CAAC,oBAAoB;gBAC9C,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,GAAG,CAAC;gBAC/C,CAAC,CAAC,eAAe,CAAC;QACtB,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,iBAAiB,CAAC,WAAW,GAAG,mBAAmB,EAAE,EAAE;YAC9D,MAAM,CAAC,IAAI,CACT,2BACE,IAAI,CAAC,iBAAiB,CAAC,WACzB,8BAA8B,mBAAmB,EAAE,+BAA+B,CACnF,CAAC;SACH;QAED,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE;YACvB,gBAAgB,GAAG,mBAAmB,EAAE,CAAC;YAEzC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC;YAEtD,IAAI,eAAe,KAAK,CAAC,EAAE;gBACzB,MAAM,IAAA,4BAAgB,EAAC,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;gBAC9D,SAAS;aACV;YAED,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,iBAAiB;gBACpD,CAAC,CAAC,IAAI,CAAC,gBAAgB;gBACvB,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC;YAE/B,IACE,IAAI,CAAC,eAAe,CAAC,QAAQ,GAAG,eAAe;gBAC/C,gBAAgB,GAAG,YAAY,EAC/B;gBACA,MAAM,IAAA,iBAAK,EAAC,CAAC,CAAC,CAAC;gBACf,SAAS;aACV;YAED,IACE,IAAI,CAAC,aAAa;gBAClB,gBAAgB,IAAI,IAAI,CAAC,iBAAiB,CAAC,WAAW,EACtD;gBACA,MAAM,aAAa,GAAG,gBAAgB,GAAG,yBAAyB,CAAC;gBACnE,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CACvC,gBAAgB,EAChB,aAAa,CACd,CAAC;gBAEF,IAAI;oBACF,MAAM,UAAU,GACd,MAAM,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAClD,gBAAgB,EAChB,aAAa,EACb,eAAe,CAChB,CAAC;oBAEJ,IAAI,gBAAgB,KAAK,mBAAmB,EAAE,EAAE;wBAC9C,MAAM,CAAC,KAAK,CACV,gEAAgE,CACjE,CAAC;wBACF,SAAS;qBACV;oBAED,IACE,UAAU;wBACV,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,gBAAgB,CAAC,EACvD;wBACA,IAAI,EAAE,WAAW,EAAE,GAAG,UAAU,CAAC;wBAEjC,WAAW,GAAG,WAAW;6BACtB,MAAM,CAAC,YAAY,CAAC;6BACpB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;wBACzB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;4BAC5B,iFAAiF;4BACjF,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CACtC,EAAE,EACF,IAAI,CAAC,GAAG,CACN,aAAa,GAAG,CAAC,EACjB,UAAU,CAAC,SAAS,CAAC,mBAAmB,CACzC,CACF,CAAC;yBACH;6BAAM;4BACL,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,WAAW,CAAC,MAAM,EAClB,IAAI,CAAC,eAAe,CAAC,QAAQ,CAC9B,CAAC;4BACF,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;4BAC3D,MAAM,kBAAkB,GACtB,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;4BAC3C,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CACtC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;yBACH;wBACD,SAAS,CAAC,4BAA4B;qBACvC;oBACD,iCAAiC;iBAClC;gBAAC,OAAO,CAAC,EAAE;oBACV,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBACvD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;iBACrD;aACF;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CACvC,gBAAgB,EAChB,eAAe,CAChB,CAAC;YAEF,MAAM,eAAe,GACnB,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;gBAC7D,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,gBAAgB,CAAC;gBAChD,CAAC,CAAC,IAAA,cAAK,EAAC,gBAAgB,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;YAE7C,MAAM,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;YACpE,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CACtC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;SACH;IACH,CAAC;IAEO,qBAAqB,CAC3B,kBAA4B,EAC5B,cAAwB;QAExB,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,kBAAkB,EAAE,GAAG,cAAc,CAAC,CAAC;IAC5D,CAAC;IACO,kBAAkB,CAAC,kBAA4B;QACrD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,EAAE;YACpD,OAAO,kBAAkB,CAAC;SAC3B;QAED,MAAM,YAAY,GAAG,IAAA,8BAAkB,EACrC,IAAI,CAAC,YAAY,EACjB,kBAAkB,CACnB,CAAC;QAEF,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAC7C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,CAC3C,CAAC;QACF,IAAI,cAAc,CAAC,MAAM,EAAE;YACzB,MAAM,CAAC,IAAI,CAAC,qBAAqB,cAAc,EAAE,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,YAAY,GAAG,IAAA,gBAAO,EAAC,IAAI,CAAC,YAAY,EAAE,GAAG,cAAc,CAAC,CAAC;QAClE,OAAO,YAAY,CAAC;IACtB,CAAC;IAEO,kBAAkB,CACxB,gBAAwB,EACxB,eAAuB;QAEvB,IAAI,cAAc,GAAG,gBAAgB,GAAG,eAAe,GAAG,CAAC,CAAC;QAE5D,IAAI,cAAc,GAAG,IAAI,CAAC,qBAAqB,EAAE;YAC/C,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;gBACrC,IAAI,cAAc,IAAI,IAAI,CAAC,gBAAgB,EAAE;oBAC3C,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC;iBACxC;aACF;iBAAM;gBACL,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC;aAC7C;SACF;QACD,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,oBAAoB,CAC1B,UAAmC,EACnC,gBAAyB;QAEzB,IAAI,UAAU,KAAK,SAAS,EAAE;YAC5B,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;YAE3C,IACE,QAAQ,CAAC,WAAW,KAAK,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE;gBAClD,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,EACpD;gBACA,MAAM,CAAC,KAAK,CACV,+KAA+K,CAChL,CAAC;gBACF,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;gBACtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;oBACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;iBAClC,CAAC,CAAC;gBACH,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;gBACpD,OAAO,KAAK,CAAC;aACd;YAED,IAAI,gBAAgB,KAAK,SAAS,EAAE;gBAClC,IAAI,QAAQ,CAAC,mBAAmB,GAAG,gBAAgB,EAAE;oBACnD,MAAM,CAAC,IAAI,CACT,kEAAkE,CACnE,CAAC;oBACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;oBACpD,OAAO,KAAK,CAAC;iBACd;aACF;YACD,OAAO,IAAI,CAAC;SACb;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,WAAmB;QACrC,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;QACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,0BAA0B,CAAC,WAAmB;QAClD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;CACF,CAAA;AApUC;IAAC,IAAA,mBAAQ,EAAC,qBAAqB,CAAC;;;;mDAS/B;AAGK;IADL,IAAA,mBAAQ,EAAC,mBAAmB,CAAC;;;;yDA0B7B;AAGK;IADL,IAAA,mBAAQ,EAAC,mBAAmB,CAAC;;;;oDAwB7B;AAvPU,YAAY;IADxB,IAAA,mBAAU,GAAE;IAcR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;IAC1B,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAHP,sBAAU;QACV,sBAAU;QACe,iCAAe,UAGjC,sCAAiB;QAClB,qCAAgB;QACR,oDAAwB;QACpC,6BAAa;QACR,4BAAiB;GApBnC,YAAY,CA4fxB;AA5fY,oCAAY","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval, SchedulerRegistry } from '@nestjs/schedule';\n\nimport {\n isCustomDs,\n EthereumHandlerKind,\n EthereumLogFilter,\n SubqlEthereumProcessorOptions,\n EthereumTransactionFilter,\n} from '@subql/common-ethereum';\nimport {\n ApiService,\n cleanedBatchBlocks,\n checkMemoryUsage,\n delay,\n getLogger,\n IndexerEvent,\n NodeConfig,\n transformBypassBlocks,\n waitForBatchSize,\n} from '@subql/node-core';\nimport {\n ApiWrapper,\n DictionaryQueryCondition,\n DictionaryQueryEntry,\n} from '@subql/types-ethereum';\nimport { MetaData } from '@subql/utils';\nimport { groupBy, range, sortBy, uniqBy, without } from 'lodash';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { calcInterval } from '../ethereum/utils.ethereum';\nimport { eventToTopic, functionToSighash } from '../utils/string';\nimport { IEthereumBlockDispatcher } from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\nconst logger = getLogger('fetch');\nlet BLOCK_TIME_VARIANCE = 5000;\nconst DICTIONARY_MAX_QUERY_SIZE = 10000;\nconst CHECK_MEMORY_INTERVAL = 60000;\nconst MINIMUM_BATCH_SIZE = 5;\nconst INTERVAL_PERCENT = 0.9;\nconst QUERY_ADDRESS_LIMIT = 50;\n\nfunction eventFilterToQueryEntry(\n filter: EthereumLogFilter,\n dsOptions: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[],\n): DictionaryQueryEntry {\n const conditions: DictionaryQueryCondition[] = [];\n\n if (Array.isArray(dsOptions)) {\n const addresses = dsOptions.map((option) => option.address).filter(Boolean);\n\n if (addresses.length !== 0 && addresses.length <= QUERY_ADDRESS_LIMIT) {\n conditions.push({\n field: 'address',\n value: addresses,\n matcher: 'inInsensitive',\n });\n }\n } else {\n if (dsOptions?.address) {\n conditions.push({\n field: 'address',\n value: dsOptions.address.toLowerCase(),\n // matcher: 'equals',\n });\n }\n }\n if (filter.topics) {\n for (let i = 0; i < Math.min(filter.topics.length, 4); i++) {\n const topic = filter.topics[i];\n if (!topic) {\n continue;\n }\n const field = `topics${i}`;\n conditions.push({\n field,\n value: eventToTopic(topic),\n matcher: 'equalTo',\n });\n }\n }\n return {\n entity: 'evmLogs',\n conditions,\n };\n}\n\nfunction callFilterToQueryEntry(\n filter: EthereumTransactionFilter,\n): DictionaryQueryEntry {\n const conditions: DictionaryQueryCondition[] = [];\n if (filter.from) {\n conditions.push({\n field: 'from',\n value: filter.from.toLowerCase(),\n });\n }\n if (filter.to) {\n conditions.push({\n field: 'to',\n value: filter.to.toLowerCase(),\n });\n }\n if (filter.function) {\n conditions.push({\n field: 'func',\n value: functionToSighash(filter.function),\n matcher: 'equalTo',\n });\n }\n return {\n entity: 'evmTransactions',\n conditions,\n };\n}\n\n@Injectable()\nexport class FetchService implements OnApplicationShutdown {\n private latestBestHeight: number;\n private latestFinalizedHeight: number;\n private isShutdown = false;\n private batchSizeScale: number;\n private templateDynamicDatasouces: SubqlProjectDs[];\n private dictionaryGenesisMatches = true;\n private evmChainId: string;\n private bypassBlocks: number[] = [];\n\n constructor(\n private apiService: ApiService,\n private nodeConfig: NodeConfig,\n @Inject('ISubqueryProject') private project: SubqueryProject,\n @Inject('IBlockDispatcher')\n private blockDispatcher: IEthereumBlockDispatcher,\n private dictionaryService: DictionaryService,\n private dynamicDsService: DynamicDsService,\n private unfinalizedBlocksService: UnfinalizedBlocksService,\n private eventEmitter: EventEmitter2,\n private schedulerRegistry: SchedulerRegistry,\n ) {\n this.batchSizeScale = 1;\n }\n\n onApplicationShutdown(): void {\n this.isShutdown = true;\n }\n\n get api(): ApiWrapper {\n return this.apiService.api;\n }\n\n async syncDynamicDatascourcesFromMeta(): Promise<void> {\n this.templateDynamicDatasouces =\n await this.dynamicDsService.getDynamicDatasources();\n }\n\n buildDictionaryQueryEntries(startBlock: number): DictionaryQueryEntry[] {\n const queryEntries: DictionaryQueryEntry[] = [];\n\n type GroupedSubqlProjectDs = SubqlProjectDs & {\n groupedOptions?: SubqlEthereumProcessorOptions[];\n };\n\n const groupdDynamicDs: GroupedSubqlProjectDs[] = Object.values(\n groupBy(this.templateDynamicDatasouces, (ds) => ds.name),\n ).map((grouped: SubqlProjectDs[]) => {\n const options = grouped.map((ds) => ds.options);\n const ref = grouped[0];\n\n return {\n ...ref,\n groupedOptions: options,\n };\n });\n\n // Only run the ds that is equal or less than startBlock\n // sort array from lowest ds.startBlock to highest\n const filteredDs: GroupedSubqlProjectDs[] = this.project.dataSources\n .concat(groupdDynamicDs)\n .filter((ds) => ds.startBlock <= startBlock)\n .sort((a, b) => a.startBlock - b.startBlock);\n\n for (const ds of filteredDs) {\n for (const handler of ds.mapping.handlers) {\n // No filters, cant use dictionary\n if (!handler.filter) return [];\n\n switch (handler.kind) {\n case EthereumHandlerKind.Block:\n return [];\n case EthereumHandlerKind.Call: {\n const filter = handler.filter as EthereumTransactionFilter;\n if (\n filter.from !== undefined ||\n filter.to !== undefined ||\n filter.function\n ) {\n queryEntries.push(callFilterToQueryEntry(filter));\n } else {\n return [];\n }\n break;\n }\n case EthereumHandlerKind.Event: {\n const filter = handler.filter as EthereumLogFilter;\n if (ds.groupedOptions) {\n queryEntries.push(\n eventFilterToQueryEntry(filter, ds.groupedOptions),\n );\n } else if (ds.options?.address || filter.topics) {\n queryEntries.push(eventFilterToQueryEntry(filter, ds.options));\n } else {\n return [];\n }\n break;\n }\n default:\n }\n }\n }\n\n return uniqBy(\n queryEntries,\n (item) =>\n `${item.entity}|${JSON.stringify(\n sortBy(item.conditions, (c) => c.field),\n )}`,\n );\n }\n\n updateDictionary(): void {\n this.dictionaryService.buildDictionaryEntryMap<SubqlProjectDs>(\n this.project.dataSources.concat(this.templateDynamicDatasouces),\n this.buildDictionaryQueryEntries.bind(this),\n );\n }\n\n private get useDictionary(): boolean {\n return (\n !!this.project.network.dictionary &&\n this.dictionaryGenesisMatches &&\n !!this.dictionaryService.getDictionaryQueryEntries(\n this.blockDispatcher.latestBufferedHeight ??\n Math.min(...this.project.dataSources.map((ds) => ds.startBlock)),\n ).length\n );\n }\n\n async init(startHeight: number): Promise<void> {\n if (this.project.network?.bypassBlocks !== undefined) {\n this.bypassBlocks = transformBypassBlocks(\n this.project.network.bypassBlocks,\n ).filter((blk) => blk >= startHeight);\n }\n if (this.api) {\n const CHAIN_INTERVAL = calcInterval(this.api) * INTERVAL_PERCENT;\n\n BLOCK_TIME_VARIANCE = Math.min(BLOCK_TIME_VARIANCE, CHAIN_INTERVAL);\n\n this.schedulerRegistry.addInterval(\n 'getLatestBlockHead',\n setInterval(() => void this.getBestBlockHead(), BLOCK_TIME_VARIANCE),\n );\n }\n await this.syncDynamicDatascourcesFromMeta();\n\n this.updateDictionary();\n this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n value: Number(this.useDictionary),\n });\n\n if (this.project.network.dictionary) {\n this.evmChainId = await this.dictionaryService.getEvmChainId();\n }\n\n await this.getFinalizedBlockHead();\n await this.getBestBlockHead();\n\n // Call metadata here, other network should align with this\n // For substrate, we might use the specVersion metadata in future if we have same error handling as in node-core\n const metadata = await this.dictionaryService.getMetadata();\n\n const validChecker = this.dictionaryValidation(metadata);\n\n if (validChecker) {\n this.dictionaryService.setDictionaryStartHeight(\n metadata._metadata.startHeight,\n );\n }\n\n await this.blockDispatcher.init(this.resetForNewDs.bind(this));\n void this.startLoop(startHeight);\n }\n\n getUseDictionary(): boolean {\n return this.useDictionary;\n }\n\n getLatestFinalizedHeight(): number {\n return this.latestFinalizedHeight;\n }\n\n @Interval(CHECK_MEMORY_INTERVAL)\n checkBatchScale(): void {\n if (this.nodeConfig['scale-batch-size']) {\n const scale = checkMemoryUsage(this.batchSizeScale, this.nodeConfig);\n\n if (this.batchSizeScale !== scale) {\n this.batchSizeScale = scale;\n }\n }\n }\n\n @Interval(BLOCK_TIME_VARIANCE)\n async getFinalizedBlockHead(): Promise<void> {\n if (!this.api) {\n logger.debug(`Skip fetch finalized block until API is ready`);\n return;\n }\n try {\n const currentFinalizedHeight = await this.api.getFinalizedBlockHeight();\n logger.debug(`finalized:${currentFinalizedHeight.toString()}`);\n const finalizedHeader = await this.api.getBlockByHeightOrHash(\n currentFinalizedHeight,\n );\n if (this.latestFinalizedHeight !== currentFinalizedHeight) {\n this.latestFinalizedHeight = currentFinalizedHeight;\n this.unfinalizedBlocksService.registerFinalizedBlock(finalizedHeader);\n if (!this.nodeConfig.unfinalizedBlocks) {\n if (!this.nodeConfig.unfinalizedBlocks) {\n this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n height: this.latestFinalizedHeight,\n });\n }\n }\n }\n } catch (e) {\n logger.warn(e, `Having a problem when get finalized block`);\n }\n }\n\n @Interval(BLOCK_TIME_VARIANCE)\n async getBestBlockHead(): Promise<void> {\n if (!this.api) {\n logger.debug(`Skip fetch best block until API is ready`);\n return;\n }\n try {\n const currentBestHeight = await this.api.getBestBlockHeight();\n logger.debug(`best:${currentBestHeight.toString()}`);\n if (this.latestBestHeight !== currentBestHeight) {\n this.latestBestHeight = currentBestHeight;\n this.eventEmitter.emit(IndexerEvent.BlockBest, {\n height: this.latestBestHeight,\n });\n\n if (this.nodeConfig.unfinalizedBlocks) {\n this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n height: this.latestBestHeight,\n });\n }\n }\n } catch (e) {\n logger.warn(e, `Having a problem when get best block`);\n }\n }\n private async startLoop(initBlockHeight: number): Promise<void> {\n await this.fillNextBlockBuffer(initBlockHeight);\n }\n\n getModulos(): number[] {\n const modulos: number[] = [];\n for (const ds of this.project.dataSources) {\n if (isCustomDs(ds)) {\n continue;\n }\n for (const handler of ds.mapping.handlers) {\n if (\n handler.kind === EthereumHandlerKind.Block &&\n handler.filter &&\n handler.filter.modulo\n ) {\n modulos.push(handler.filter.modulo);\n }\n }\n }\n return modulos;\n }\n\n getModuloBlocks(startHeight: number, endHeight: number): number[] {\n const modulos = this.getModulos();\n const moduloBlocks: number[] = [];\n for (let i = startHeight; i < endHeight; i++) {\n if (modulos.find((m) => i % m === 0)) {\n moduloBlocks.push(i);\n }\n }\n return moduloBlocks;\n }\n\n getEnqueuedModuloBlocks(startBlockHeight: number): number[] {\n return this.getModuloBlocks(\n startBlockHeight,\n this.nodeConfig.batchSize * Math.max(...this.getModulos()) +\n startBlockHeight,\n ).slice(0, this.nodeConfig.batchSize);\n }\n\n async fillNextBlockBuffer(initBlockHeight: number): Promise<void> {\n let startBlockHeight: number;\n let scaledBatchSize: number;\n const handlers = [].concat(\n ...this.project.dataSources.map((ds) => ds.mapping.handlers),\n );\n\n const getStartBlockHeight = (): number => {\n return this.blockDispatcher.latestBufferedHeight\n ? this.blockDispatcher.latestBufferedHeight + 1\n : initBlockHeight;\n };\n\n if (this.dictionaryService.startHeight > getStartBlockHeight()) {\n logger.warn(\n `Dictionary start height ${\n this.dictionaryService.startHeight\n } is beyond indexing height ${getStartBlockHeight()}, skipping dictionary for now`,\n );\n }\n\n while (!this.isShutdown) {\n startBlockHeight = getStartBlockHeight();\n\n scaledBatchSize = this.blockDispatcher.smartBatchSize;\n\n if (scaledBatchSize === 0) {\n await waitForBatchSize(this.blockDispatcher.minimumHeapLimit);\n continue;\n }\n\n const latestHeight = this.nodeConfig.unfinalizedBlocks\n ? this.latestBestHeight\n : this.latestFinalizedHeight;\n\n if (\n this.blockDispatcher.freeSize < scaledBatchSize ||\n startBlockHeight > latestHeight\n ) {\n await delay(1);\n continue;\n }\n\n if (\n this.useDictionary &&\n startBlockHeight >= this.dictionaryService.startHeight\n ) {\n const queryEndBlock = startBlockHeight + DICTIONARY_MAX_QUERY_SIZE;\n const moduloBlocks = this.getModuloBlocks(\n startBlockHeight,\n queryEndBlock,\n );\n\n try {\n const dictionary =\n await this.dictionaryService.scopedDictionaryEntries(\n startBlockHeight,\n queryEndBlock,\n scaledBatchSize,\n );\n\n if (startBlockHeight !== getStartBlockHeight()) {\n logger.debug(\n `Queue was reset for new DS, discarding dictionary query result`,\n );\n continue;\n }\n\n if (\n dictionary &&\n this.dictionaryValidation(dictionary, startBlockHeight)\n ) {\n let { batchBlocks } = dictionary;\n\n batchBlocks = batchBlocks\n .concat(moduloBlocks)\n .sort((a, b) => a - b);\n if (batchBlocks.length === 0) {\n // There we're no blocks in this query range, we can set a new height we're up to\n await this.blockDispatcher.enqueueBlocks(\n [],\n Math.min(\n queryEndBlock - 1,\n dictionary._metadata.lastProcessedHeight,\n ),\n );\n } else {\n const maxBlockSize = Math.min(\n batchBlocks.length,\n this.blockDispatcher.freeSize,\n );\n const enqueuingBlocks = batchBlocks.slice(0, maxBlockSize);\n const cleanedBatchBlocks =\n this.filteredBlockBatch(enqueuingBlocks);\n await this.blockDispatcher.enqueueBlocks(\n cleanedBatchBlocks,\n this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n );\n }\n continue; // skip nextBlockRange() way\n }\n // else use this.nextBlockRange()\n } catch (e) {\n logger.debug(`Fetch dictionary stopped: ${e.message}`);\n this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n }\n }\n\n const endHeight = this.nextEndBlockHeight(\n startBlockHeight,\n scaledBatchSize,\n );\n\n const enqueuingBlocks =\n handlers.length && this.getModulos().length === handlers.length\n ? this.getEnqueuedModuloBlocks(startBlockHeight)\n : range(startBlockHeight, endHeight + 1);\n\n const cleanedBatchBlocks = this.filteredBlockBatch(enqueuingBlocks);\n await this.blockDispatcher.enqueueBlocks(\n cleanedBatchBlocks,\n this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n );\n }\n }\n\n private getLatestBufferHeight(\n cleanedBatchBlocks: number[],\n rawBatchBlocks: number[],\n ): number {\n return Math.max(...cleanedBatchBlocks, ...rawBatchBlocks);\n }\n private filteredBlockBatch(currentBatchBlocks: number[]): number[] {\n if (!this.bypassBlocks.length || !currentBatchBlocks) {\n return currentBatchBlocks;\n }\n\n const cleanedBatch = cleanedBatchBlocks(\n this.bypassBlocks,\n currentBatchBlocks,\n );\n\n const pollutedBlocks = this.bypassBlocks.filter(\n (b) => b < Math.max(...currentBatchBlocks),\n );\n if (pollutedBlocks.length) {\n logger.info(`Bypassing blocks: ${pollutedBlocks}`);\n }\n this.bypassBlocks = without(this.bypassBlocks, ...pollutedBlocks);\n return cleanedBatch;\n }\n\n private nextEndBlockHeight(\n startBlockHeight: number,\n scaledBatchSize: number,\n ): number {\n let endBlockHeight = startBlockHeight + scaledBatchSize - 1;\n\n if (endBlockHeight > this.latestFinalizedHeight) {\n if (this.nodeConfig.unfinalizedBlocks) {\n if (endBlockHeight >= this.latestBestHeight) {\n endBlockHeight = this.latestBestHeight;\n }\n } else {\n endBlockHeight = this.latestFinalizedHeight;\n }\n }\n return endBlockHeight;\n }\n\n private dictionaryValidation(\n dictionary: { _metadata: MetaData },\n startBlockHeight?: number,\n ): boolean {\n if (dictionary !== undefined) {\n const { _metadata: metaData } = dictionary;\n\n if (\n metaData.genesisHash !== this.api.getGenesisHash() &&\n this.evmChainId !== this.api.getChainId().toString()\n ) {\n logger.error(\n 'The dictionary that you have specified does not match the chain you are indexing, it will be ignored. Please update your project manifest to reference the correct dictionary',\n );\n this.dictionaryGenesisMatches = false;\n this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n value: Number(this.useDictionary),\n });\n this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n return false;\n }\n\n if (startBlockHeight !== undefined) {\n if (metaData.lastProcessedHeight < startBlockHeight) {\n logger.warn(\n `Dictionary indexed block is behind current indexing block height`,\n );\n this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n return false;\n }\n }\n return true;\n }\n return false;\n }\n\n async resetForNewDs(blockHeight: number): Promise<void> {\n await this.syncDynamicDatascourcesFromMeta();\n this.dynamicDsService.deleteTempDsRecords(blockHeight);\n this.updateDictionary();\n this.blockDispatcher.flushQueue(blockHeight);\n }\n\n async resetForIncorrectBestBlock(blockHeight: number): Promise<void> {\n await this.syncDynamicDatascourcesFromMeta();\n this.updateDictionary();\n this.blockDispatcher.flushQueue(blockHeight);\n }\n}\n"]}
1
+ {"version":3,"file":"fetch.service.js","sourceRoot":"","sources":["../../src/indexer/fetch.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAA2E;AAC3E,yDAAsD;AACtD,+CAA+D;AAE/D,4DAMgC;AAChC,gDAU0B;AAO1B,mCAAiE;AACjE,kEAA+E;AAC/E,+DAA0D;AAC1D,4CAAkE;AAElE,6DAAyD;AACzD,6DAAwD;AACxD,2EAAuE;AAEvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,OAAO,CAAC,CAAC;AAClC,IAAI,mBAAmB,GAAG,IAAI,CAAC;AAC/B,MAAM,yBAAyB,GAAG,KAAK,CAAC;AACxC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,SAAS,uBAAuB,CAC9B,MAAyB,EACzB,SAA0E;IAE1E,MAAM,UAAU,GAA+B,EAAE,CAAC;IAElD,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;QAC5B,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE5E,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,mBAAmB,EAAE;YACrE,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,eAAe;aACzB,CAAC,CAAC;SACJ;KACF;SAAM;QACL,IAAI,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,OAAO,EAAE;YACtB,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE;gBACtC,qBAAqB;aACtB,CAAC,CAAC;SACJ;KACF;IACD,IAAI,MAAM,CAAC,MAAM,EAAE;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,EAAE;gBACV,SAAS;aACV;YACD,MAAM,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK;gBACL,KAAK,EAAE,IAAA,qBAAY,EAAC,KAAK,CAAC;gBAC1B,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;SACJ;KACF;IACD,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,MAAiC;IAEjC,MAAM,UAAU,GAA+B,EAAE,CAAC;IAClD,IAAI,MAAM,CAAC,IAAI,EAAE;QACf,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE;SACjC,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,EAAE,EAAE;QACb,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE;SAC/B,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,QAAQ,EAAE;QACnB,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,IAAA,0BAAiB,EAAC,MAAM,CAAC,QAAQ,CAAC;YACzC,OAAO,EAAE,SAAS;SACnB,CAAC,CAAC;KACJ;IACD,OAAO;QACL,MAAM,EAAE,iBAAiB;QACzB,UAAU;KACX,CAAC;AACJ,CAAC;AAGD,IAAa,YAAY,GAAzB,MAAa,YAAY;IAUvB,YACU,UAAsB,EACtB,UAAsB,EACM,OAAwB,EAEpD,eAAyC,EACzC,iBAAoC,EACpC,gBAAkC,EAClC,wBAAkD,EAClD,YAA2B,EAC3B,iBAAoC;QATpC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QAEpD,oBAAe,GAAf,eAAe,CAA0B;QACzC,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,iBAAY,GAAZ,YAAY,CAAe;QAC3B,sBAAiB,GAAjB,iBAAiB,CAAmB;QAjBtC,eAAU,GAAG,KAAK,CAAC;QAGnB,wBAAmB,GAAG,KAAK,CAAC;QAE5B,iBAAY,GAAa,EAAE,CAAC;QAclC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,qBAAqB;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,+BAA+B;QACnC,IAAI,CAAC,yBAAyB;YAC5B,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;IACxD,CAAC;IAED,2BAA2B,CAAC,UAAkB;;QAC5C,MAAM,YAAY,GAA2B,EAAE,CAAC;QAMhD,MAAM,eAAe,GAA4B,MAAM,CAAC,MAAM,CAC5D,IAAA,gBAAO,EAAC,IAAI,CAAC,yBAAyB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CACzD,CAAC,GAAG,CAAC,CAAC,OAAyB,EAAE,EAAE;YAClC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YAChD,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvB,uCACK,GAAG,KACN,cAAc,EAAE,OAAO,IACvB;QACJ,CAAC,CAAC,CAAC;QAEH,wDAAwD;QACxD,kDAAkD;QAClD,MAAM,UAAU,GAA4B,IAAI,CAAC,OAAO,CAAC,WAAW;aACjE,MAAM,CAAC,eAAe,CAAC;aACvB,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,UAAU,CAAC;aAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAE/C,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE;YAC3B,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,kCAAkC;gBAClC,IAAI,CAAC,OAAO,CAAC,MAAM;oBAAE,OAAO,EAAE,CAAC;gBAE/B,QAAQ,OAAO,CAAC,IAAI,EAAE;oBACpB,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,EAAE,CAAC;oBACZ,KAAK,qCAAmB,CAAC,IAAI,CAAC,CAAC;wBAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAmC,CAAC;wBAC3D,IACE,MAAM,CAAC,IAAI,KAAK,SAAS;4BACzB,MAAM,CAAC,EAAE,KAAK,SAAS;4BACvB,MAAM,CAAC,QAAQ,EACf;4BACA,YAAY,CAAC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC;yBACnD;6BAAM;4BACL,OAAO,EAAE,CAAC;yBACX;wBACD,MAAM;qBACP;oBACD,KAAK,qCAAmB,CAAC,KAAK,CAAC,CAAC;wBAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,MAA2B,CAAC;wBACnD,IAAI,EAAE,CAAC,cAAc,EAAE;4BACrB,YAAY,CAAC,IAAI,CACf,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC,CACnD,CAAC;yBACH;6BAAM,IAAI,CAAA,MAAA,EAAE,CAAC,OAAO,0CAAE,OAAO,KAAI,MAAM,CAAC,MAAM,EAAE;4BAC/C,YAAY,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;yBAChE;6BAAM;4BACL,OAAO,EAAE,CAAC;yBACX;wBACD,MAAM;qBACP;oBACD,QAAQ;iBACT;aACF;SACF;QAED,OAAO,IAAA,eAAM,EACX,YAAY,EACZ,CAAC,IAAI,EAAE,EAAE,CACP,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAC9B,IAAA,eAAM,EAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACxC,EAAE,CACN,CAAC;IACJ,CAAC;IAED,gBAAgB;QACd,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAC5C,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,EAC/D,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAC5C,CAAC;IACJ,CAAC;IAED,IAAY,aAAa;;QACvB,OAAO,CACL,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU;YACjC,IAAI,CAAC,mBAAmB;YACxB,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,yBAAyB,CAChD,MAAA,IAAI,CAAC,eAAe,CAAC,oBAAoB,mCACvC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CACnE,CAAC,MAAM,CACT,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,WAAmB;;QAC5B,IAAI,CAAA,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,0CAAE,YAAY,MAAK,SAAS,EAAE;YACpD,IAAI,CAAC,YAAY,GAAG,IAAA,iCAAqB,EACvC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAClC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,WAAW,CAAC,CAAC;SACvC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE;YACZ,MAAM,cAAc,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,GAAG,CAAC,GAAG,gBAAgB,CAAC;YAEjE,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;YAEpE,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAChC,oBAAoB,EACpB,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,gBAAgB,EAAE,EAAE,mBAAmB,CAAC,CACrE,CAAC;SACH;QACD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAE7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,4DAA4D;QAC5D,iHAAiH;QACjH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,CAAC;QAC5D,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE;YACnC,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;SAChE;QACD,MAAM,eAAe,GAAG,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QAE5D,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;QAE3E,IAAI,eAAe,EAAE;YACnB,IAAI,CAAC,iBAAiB,CAAC,wBAAwB,CAC7C,MAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,SAAS,0CAAE,WAAW,CACjC,CAAC;SACH;QAED,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,wBAAwB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAGD,eAAe;QACb,IAAI,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE;YACvC,MAAM,KAAK,GAAG,IAAA,4BAAgB,EAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAErE,IAAI,IAAI,CAAC,cAAc,KAAK,KAAK,EAAE;gBACjC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;aAC7B;SACF;IACH,CAAC;IAGD,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC9D,OAAO;SACR;QACD,IAAI;YACF,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,aAAa,sBAAsB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC/D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC3D,sBAAsB,CACvB,CAAC;YACF,IAAI,IAAI,CAAC,qBAAqB,KAAK,sBAAsB,EAAE;gBACzD,IAAI,CAAC,qBAAqB,GAAG,sBAAsB,CAAC;gBACpD,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,eAAe,CAAC,CAAC;gBACtE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;oBACtC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;wBACtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;4BAC/C,MAAM,EAAE,IAAI,CAAC,qBAAqB;yBACnC,CAAC,CAAC;qBACJ;iBACF;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,2CAA2C,CAAC,CAAC;SAC7D;IACH,CAAC;IAGD,KAAK,CAAC,gBAAgB;QACpB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,OAAO;SACR;QACD,IAAI;YACF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC9D,MAAM,CAAC,KAAK,CAAC,QAAQ,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACrD,IAAI,IAAI,CAAC,gBAAgB,KAAK,iBAAiB,EAAE;gBAC/C,IAAI,CAAC,gBAAgB,GAAG,iBAAiB,CAAC;gBAC1C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,SAAS,EAAE;oBAC7C,MAAM,EAAE,IAAI,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;gBAEH,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;oBACrC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;wBAC/C,MAAM,EAAE,IAAI,CAAC,gBAAgB;qBAC9B,CAAC,CAAC;iBACJ;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,sCAAsC,CAAC,CAAC;SACxD;IACH,CAAC;IACO,KAAK,CAAC,SAAS,CAAC,eAAuB;QAC7C,MAAM,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IAClD,CAAC;IAED,UAAU;QACR,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;YACzC,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;gBAClB,SAAS;aACV;YACD,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IACE,OAAO,CAAC,IAAI,KAAK,qCAAmB,CAAC,KAAK;oBAC1C,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,MAAM,CAAC,MAAM,EACrB;oBACA,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;iBACrC;aACF;SACF;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,eAAe,CAAC,WAAmB,EAAE,SAAiB;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,WAAW,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE;YAC5C,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE;gBACpC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACtB;SACF;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,uBAAuB,CAAC,gBAAwB;QAC9C,OAAO,IAAI,CAAC,eAAe,CACzB,gBAAgB,EAChB,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACxD,gBAAgB,CACnB,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,eAAuB;QAC/C,IAAI,gBAAwB,CAAC;QAC7B,IAAI,eAAuB,CAAC;QAC5B,MAAM,QAAQ,GAAG,EAAE,CAAC,MAAM,CACxB,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC7D,CAAC;QAEF,MAAM,mBAAmB,GAAG,GAAW,EAAE;YACvC,OAAO,IAAI,CAAC,eAAe,CAAC,oBAAoB;gBAC9C,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,GAAG,CAAC;gBAC/C,CAAC,CAAC,eAAe,CAAC;QACtB,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,iBAAiB,CAAC,WAAW,GAAG,mBAAmB,EAAE,EAAE;YAC9D,MAAM,CAAC,IAAI,CACT,2BACE,IAAI,CAAC,iBAAiB,CAAC,WACzB,8BAA8B,mBAAmB,EAAE,+BAA+B,CACnF,CAAC;SACH;QAED,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE;YACvB,gBAAgB,GAAG,mBAAmB,EAAE,CAAC;YAEzC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC;YAEtD,IAAI,eAAe,KAAK,CAAC,EAAE;gBACzB,MAAM,IAAA,4BAAgB,EAAC,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;gBAC9D,SAAS;aACV;YAED,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,iBAAiB;gBACpD,CAAC,CAAC,IAAI,CAAC,gBAAgB;gBACvB,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC;YAE/B,IACE,IAAI,CAAC,eAAe,CAAC,QAAQ,GAAG,eAAe;gBAC/C,gBAAgB,GAAG,YAAY,EAC/B;gBACA,MAAM,IAAA,iBAAK,EAAC,CAAC,CAAC,CAAC;gBACf,SAAS;aACV;YAED,IACE,IAAI,CAAC,aAAa;gBAClB,gBAAgB,IAAI,IAAI,CAAC,iBAAiB,CAAC,WAAW,EACtD;gBACA,MAAM,aAAa,GAAG,gBAAgB,GAAG,yBAAyB,CAAC;gBACnE,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CACvC,gBAAgB,EAChB,aAAa,CACd,CAAC;gBAEF,IAAI;oBACF,MAAM,UAAU,GACd,MAAM,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAClD,gBAAgB,EAChB,aAAa,EACb,eAAe,CAChB,CAAC;oBAEJ,IAAI,gBAAgB,KAAK,mBAAmB,EAAE,EAAE;wBAC9C,MAAM,CAAC,KAAK,CACV,gEAAgE,CACjE,CAAC;wBACF,SAAS;qBACV;oBAED,IACE,UAAU;wBACV,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,gBAAgB,CAAC,EACvD;wBACA,IAAI,EAAE,WAAW,EAAE,GAAG,UAAU,CAAC;wBAEjC,WAAW,GAAG,WAAW;6BACtB,MAAM,CAAC,YAAY,CAAC;6BACpB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;wBACzB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;4BAC5B,iFAAiF;4BACjF,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CACtC,EAAE,EACF,IAAI,CAAC,GAAG,CACN,aAAa,GAAG,CAAC,EACjB,UAAU,CAAC,SAAS,CAAC,mBAAmB,CACzC,CACF,CAAC;yBACH;6BAAM;4BACL,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,WAAW,CAAC,MAAM,EAClB,IAAI,CAAC,eAAe,CAAC,QAAQ,CAC9B,CAAC;4BACF,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;4BAC3D,MAAM,kBAAkB,GACtB,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;4BAC3C,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CACtC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;yBACH;wBACD,SAAS,CAAC,4BAA4B;qBACvC;oBACD,iCAAiC;iBAClC;gBAAC,OAAO,CAAC,EAAE;oBACV,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBACvD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;iBACrD;aACF;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CACvC,gBAAgB,EAChB,eAAe,CAChB,CAAC;YAEF,MAAM,eAAe,GACnB,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;gBAC7D,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,gBAAgB,CAAC;gBAChD,CAAC,CAAC,IAAA,cAAK,EAAC,gBAAgB,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;YAE7C,MAAM,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;YACpE,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CACtC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;SACH;IACH,CAAC;IAEO,qBAAqB,CAC3B,kBAA4B,EAC5B,cAAwB;QAExB,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,kBAAkB,EAAE,GAAG,cAAc,CAAC,CAAC;IAC5D,CAAC;IACO,kBAAkB,CAAC,kBAA4B;QACrD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,EAAE;YACpD,OAAO,kBAAkB,CAAC;SAC3B;QAED,MAAM,YAAY,GAAG,IAAA,8BAAkB,EACrC,IAAI,CAAC,YAAY,EACjB,kBAAkB,CACnB,CAAC;QAEF,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAC7C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,CAC3C,CAAC;QACF,IAAI,cAAc,CAAC,MAAM,EAAE;YACzB,MAAM,CAAC,IAAI,CAAC,qBAAqB,cAAc,EAAE,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,YAAY,GAAG,IAAA,gBAAO,EAAC,IAAI,CAAC,YAAY,EAAE,GAAG,cAAc,CAAC,CAAC;QAClE,OAAO,YAAY,CAAC;IACtB,CAAC;IAEO,kBAAkB,CACxB,gBAAwB,EACxB,eAAuB;QAEvB,IAAI,cAAc,GAAG,gBAAgB,GAAG,eAAe,GAAG,CAAC,CAAC;QAE5D,IAAI,cAAc,GAAG,IAAI,CAAC,qBAAqB,EAAE;YAC/C,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;gBACrC,IAAI,cAAc,IAAI,IAAI,CAAC,gBAAgB,EAAE;oBAC3C,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC;iBACxC;aACF;iBAAM;gBACL,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC;aAC7C;SACF;QACD,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,oBAAoB,CAC1B,UAAmC,EACnC,gBAAyB;QAEzB,MAAM,QAAQ,GAAG,GAAY,EAAE;YAC7B,IAAI,UAAU,KAAK,SAAS,EAAE;gBAC5B,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;gBAE3C,IACE,QAAQ,CAAC,WAAW,KAAK,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE;oBAClD,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,EACpD;oBACA,MAAM,CAAC,KAAK,CACV,+KAA+K,CAChL,CAAC;oBACF,OAAO,KAAK,CAAC;iBACd;gBAED,IACE,gBAAgB,KAAK,SAAS;oBAC9B,QAAQ,CAAC,mBAAmB,GAAG,gBAAgB,EAC/C;oBACA,MAAM,CAAC,IAAI,CACT,kEAAkE,CACnE,CAAC;oBACF,OAAO,KAAK,CAAC;iBACd;gBACD,OAAO,IAAI,CAAC;aACb;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;QAEF,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;QAEzB,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;QACjC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;YACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;QAEpD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,WAAmB;QACrC,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;QACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;IACD,KAAK,CAAC,0BAA0B,CAAC,WAAmB;QAClD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;CACF,CAAA;AAzUC;IADC,IAAA,mBAAQ,EAAC,qBAAqB,CAAC;;;;mDAS/B;AAGD;IADC,IAAA,mBAAQ,EAAC,mBAAmB,CAAC;;;;yDA0B7B;AAGD;IADC,IAAA,mBAAQ,EAAC,mBAAmB,CAAC;;;;oDAwB7B;AAhPU,YAAY;IADxB,IAAA,mBAAU,GAAE;IAcR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;IAC1B,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAHP,sBAAU;QACV,sBAAU;QACe,iCAAe,UAGjC,sCAAiB;QAClB,qCAAgB;QACR,oDAAwB;QACpC,6BAAa;QACR,4BAAiB;GApBnC,YAAY,CA2fxB;AA3fY,oCAAY","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval, SchedulerRegistry } from '@nestjs/schedule';\n\nimport {\n isCustomDs,\n EthereumHandlerKind,\n EthereumLogFilter,\n SubqlEthereumProcessorOptions,\n EthereumTransactionFilter,\n} from '@subql/common-ethereum';\nimport {\n ApiService,\n cleanedBatchBlocks,\n checkMemoryUsage,\n delay,\n getLogger,\n IndexerEvent,\n NodeConfig,\n transformBypassBlocks,\n waitForBatchSize,\n} from '@subql/node-core';\nimport {\n ApiWrapper,\n DictionaryQueryCondition,\n DictionaryQueryEntry,\n} from '@subql/types-ethereum';\nimport { MetaData } from '@subql/utils';\nimport { groupBy, range, sortBy, uniqBy, without } from 'lodash';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { calcInterval } from '../ethereum/utils.ethereum';\nimport { eventToTopic, functionToSighash } from '../utils/string';\nimport { IEthereumBlockDispatcher } from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\nconst logger = getLogger('fetch');\nlet BLOCK_TIME_VARIANCE = 5000;\nconst DICTIONARY_MAX_QUERY_SIZE = 10000;\nconst CHECK_MEMORY_INTERVAL = 60000;\nconst MINIMUM_BATCH_SIZE = 5;\nconst INTERVAL_PERCENT = 0.9;\nconst QUERY_ADDRESS_LIMIT = 50;\n\nfunction eventFilterToQueryEntry(\n filter: EthereumLogFilter,\n dsOptions: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[],\n): DictionaryQueryEntry {\n const conditions: DictionaryQueryCondition[] = [];\n\n if (Array.isArray(dsOptions)) {\n const addresses = dsOptions.map((option) => option.address).filter(Boolean);\n\n if (addresses.length !== 0 && addresses.length <= QUERY_ADDRESS_LIMIT) {\n conditions.push({\n field: 'address',\n value: addresses,\n matcher: 'inInsensitive',\n });\n }\n } else {\n if (dsOptions?.address) {\n conditions.push({\n field: 'address',\n value: dsOptions.address.toLowerCase(),\n // matcher: 'equals',\n });\n }\n }\n if (filter.topics) {\n for (let i = 0; i < Math.min(filter.topics.length, 4); i++) {\n const topic = filter.topics[i];\n if (!topic) {\n continue;\n }\n const field = `topics${i}`;\n conditions.push({\n field,\n value: eventToTopic(topic),\n matcher: 'equalTo',\n });\n }\n }\n return {\n entity: 'evmLogs',\n conditions,\n };\n}\n\nfunction callFilterToQueryEntry(\n filter: EthereumTransactionFilter,\n): DictionaryQueryEntry {\n const conditions: DictionaryQueryCondition[] = [];\n if (filter.from) {\n conditions.push({\n field: 'from',\n value: filter.from.toLowerCase(),\n });\n }\n if (filter.to) {\n conditions.push({\n field: 'to',\n value: filter.to.toLowerCase(),\n });\n }\n if (filter.function) {\n conditions.push({\n field: 'func',\n value: functionToSighash(filter.function),\n matcher: 'equalTo',\n });\n }\n return {\n entity: 'evmTransactions',\n conditions,\n };\n}\n\n@Injectable()\nexport class FetchService implements OnApplicationShutdown {\n private latestBestHeight: number;\n private latestFinalizedHeight: number;\n private isShutdown = false;\n private batchSizeScale: number;\n private templateDynamicDatasouces: SubqlProjectDs[];\n private dictionaryMetaValid = false;\n private evmChainId: string;\n private bypassBlocks: number[] = [];\n\n constructor(\n private apiService: ApiService,\n private nodeConfig: NodeConfig,\n @Inject('ISubqueryProject') private project: SubqueryProject,\n @Inject('IBlockDispatcher')\n private blockDispatcher: IEthereumBlockDispatcher,\n private dictionaryService: DictionaryService,\n private dynamicDsService: DynamicDsService,\n private unfinalizedBlocksService: UnfinalizedBlocksService,\n private eventEmitter: EventEmitter2,\n private schedulerRegistry: SchedulerRegistry,\n ) {\n this.batchSizeScale = 1;\n }\n\n onApplicationShutdown(): void {\n this.isShutdown = true;\n }\n\n get api(): ApiWrapper {\n return this.apiService.api;\n }\n\n async syncDynamicDatascourcesFromMeta(): Promise<void> {\n this.templateDynamicDatasouces =\n await this.dynamicDsService.getDynamicDatasources();\n }\n\n buildDictionaryQueryEntries(startBlock: number): DictionaryQueryEntry[] {\n const queryEntries: DictionaryQueryEntry[] = [];\n\n type GroupedSubqlProjectDs = SubqlProjectDs & {\n groupedOptions?: SubqlEthereumProcessorOptions[];\n };\n\n const groupdDynamicDs: GroupedSubqlProjectDs[] = Object.values(\n groupBy(this.templateDynamicDatasouces, (ds) => ds.name),\n ).map((grouped: SubqlProjectDs[]) => {\n const options = grouped.map((ds) => ds.options);\n const ref = grouped[0];\n\n return {\n ...ref,\n groupedOptions: options,\n };\n });\n\n // Only run the ds that is equal or less than startBlock\n // sort array from lowest ds.startBlock to highest\n const filteredDs: GroupedSubqlProjectDs[] = this.project.dataSources\n .concat(groupdDynamicDs)\n .filter((ds) => ds.startBlock <= startBlock)\n .sort((a, b) => a.startBlock - b.startBlock);\n\n for (const ds of filteredDs) {\n for (const handler of ds.mapping.handlers) {\n // No filters, cant use dictionary\n if (!handler.filter) return [];\n\n switch (handler.kind) {\n case EthereumHandlerKind.Block:\n return [];\n case EthereumHandlerKind.Call: {\n const filter = handler.filter as EthereumTransactionFilter;\n if (\n filter.from !== undefined ||\n filter.to !== undefined ||\n filter.function\n ) {\n queryEntries.push(callFilterToQueryEntry(filter));\n } else {\n return [];\n }\n break;\n }\n case EthereumHandlerKind.Event: {\n const filter = handler.filter as EthereumLogFilter;\n if (ds.groupedOptions) {\n queryEntries.push(\n eventFilterToQueryEntry(filter, ds.groupedOptions),\n );\n } else if (ds.options?.address || filter.topics) {\n queryEntries.push(eventFilterToQueryEntry(filter, ds.options));\n } else {\n return [];\n }\n break;\n }\n default:\n }\n }\n }\n\n return uniqBy(\n queryEntries,\n (item) =>\n `${item.entity}|${JSON.stringify(\n sortBy(item.conditions, (c) => c.field),\n )}`,\n );\n }\n\n updateDictionary(): void {\n this.dictionaryService.buildDictionaryEntryMap<SubqlProjectDs>(\n this.project.dataSources.concat(this.templateDynamicDatasouces),\n this.buildDictionaryQueryEntries.bind(this),\n );\n }\n\n private get useDictionary(): boolean {\n return (\n !!this.project.network.dictionary &&\n this.dictionaryMetaValid &&\n !!this.dictionaryService.getDictionaryQueryEntries(\n this.blockDispatcher.latestBufferedHeight ??\n Math.min(...this.project.dataSources.map((ds) => ds.startBlock)),\n ).length\n );\n }\n\n async init(startHeight: number): Promise<void> {\n if (this.project.network?.bypassBlocks !== undefined) {\n this.bypassBlocks = transformBypassBlocks(\n this.project.network.bypassBlocks,\n ).filter((blk) => blk >= startHeight);\n }\n if (this.api) {\n const CHAIN_INTERVAL = calcInterval(this.api) * INTERVAL_PERCENT;\n\n BLOCK_TIME_VARIANCE = Math.min(BLOCK_TIME_VARIANCE, CHAIN_INTERVAL);\n\n this.schedulerRegistry.addInterval(\n 'getLatestBlockHead',\n setInterval(() => void this.getBestBlockHead(), BLOCK_TIME_VARIANCE),\n );\n }\n await this.syncDynamicDatascourcesFromMeta();\n\n this.updateDictionary();\n // Call metadata here, other network should align with this\n // For substrate, we might use the specVersion metadata in future if we have same error handling as in node-core\n const metadata = await this.dictionaryService.getMetadata();\n if (this.project.network.dictionary) {\n this.evmChainId = await this.dictionaryService.getEvmChainId();\n }\n const dictionaryValid = this.dictionaryValidation(metadata);\n\n await Promise.all([this.getFinalizedBlockHead(), this.getBestBlockHead()]);\n\n if (dictionaryValid) {\n this.dictionaryService.setDictionaryStartHeight(\n metadata?._metadata?.startHeight,\n );\n }\n\n await this.blockDispatcher.init(this.resetForNewDs.bind(this));\n void this.startLoop(startHeight);\n }\n\n getUseDictionary(): boolean {\n return this.useDictionary;\n }\n\n getLatestFinalizedHeight(): number {\n return this.latestFinalizedHeight;\n }\n\n @Interval(CHECK_MEMORY_INTERVAL)\n checkBatchScale(): void {\n if (this.nodeConfig['scale-batch-size']) {\n const scale = checkMemoryUsage(this.batchSizeScale, this.nodeConfig);\n\n if (this.batchSizeScale !== scale) {\n this.batchSizeScale = scale;\n }\n }\n }\n\n @Interval(BLOCK_TIME_VARIANCE)\n async getFinalizedBlockHead(): Promise<void> {\n if (!this.api) {\n logger.debug(`Skip fetch finalized block until API is ready`);\n return;\n }\n try {\n const currentFinalizedHeight = await this.api.getFinalizedBlockHeight();\n logger.debug(`finalized:${currentFinalizedHeight.toString()}`);\n const finalizedHeader = await this.api.getBlockByHeightOrHash(\n currentFinalizedHeight,\n );\n if (this.latestFinalizedHeight !== currentFinalizedHeight) {\n this.latestFinalizedHeight = currentFinalizedHeight;\n this.unfinalizedBlocksService.registerFinalizedBlock(finalizedHeader);\n if (!this.nodeConfig.unfinalizedBlocks) {\n if (!this.nodeConfig.unfinalizedBlocks) {\n this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n height: this.latestFinalizedHeight,\n });\n }\n }\n }\n } catch (e) {\n logger.warn(e, `Having a problem when get finalized block`);\n }\n }\n\n @Interval(BLOCK_TIME_VARIANCE)\n async getBestBlockHead(): Promise<void> {\n if (!this.api) {\n logger.debug(`Skip fetch best block until API is ready`);\n return;\n }\n try {\n const currentBestHeight = await this.api.getBestBlockHeight();\n logger.debug(`best:${currentBestHeight.toString()}`);\n if (this.latestBestHeight !== currentBestHeight) {\n this.latestBestHeight = currentBestHeight;\n this.eventEmitter.emit(IndexerEvent.BlockBest, {\n height: this.latestBestHeight,\n });\n\n if (this.nodeConfig.unfinalizedBlocks) {\n this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n height: this.latestBestHeight,\n });\n }\n }\n } catch (e) {\n logger.warn(e, `Having a problem when get best block`);\n }\n }\n private async startLoop(initBlockHeight: number): Promise<void> {\n await this.fillNextBlockBuffer(initBlockHeight);\n }\n\n getModulos(): number[] {\n const modulos: number[] = [];\n for (const ds of this.project.dataSources) {\n if (isCustomDs(ds)) {\n continue;\n }\n for (const handler of ds.mapping.handlers) {\n if (\n handler.kind === EthereumHandlerKind.Block &&\n handler.filter &&\n handler.filter.modulo\n ) {\n modulos.push(handler.filter.modulo);\n }\n }\n }\n return modulos;\n }\n\n getModuloBlocks(startHeight: number, endHeight: number): number[] {\n const modulos = this.getModulos();\n const moduloBlocks: number[] = [];\n for (let i = startHeight; i < endHeight; i++) {\n if (modulos.find((m) => i % m === 0)) {\n moduloBlocks.push(i);\n }\n }\n return moduloBlocks;\n }\n\n getEnqueuedModuloBlocks(startBlockHeight: number): number[] {\n return this.getModuloBlocks(\n startBlockHeight,\n this.nodeConfig.batchSize * Math.max(...this.getModulos()) +\n startBlockHeight,\n ).slice(0, this.nodeConfig.batchSize);\n }\n\n async fillNextBlockBuffer(initBlockHeight: number): Promise<void> {\n let startBlockHeight: number;\n let scaledBatchSize: number;\n const handlers = [].concat(\n ...this.project.dataSources.map((ds) => ds.mapping.handlers),\n );\n\n const getStartBlockHeight = (): number => {\n return this.blockDispatcher.latestBufferedHeight\n ? this.blockDispatcher.latestBufferedHeight + 1\n : initBlockHeight;\n };\n\n if (this.dictionaryService.startHeight > getStartBlockHeight()) {\n logger.warn(\n `Dictionary start height ${\n this.dictionaryService.startHeight\n } is beyond indexing height ${getStartBlockHeight()}, skipping dictionary for now`,\n );\n }\n\n while (!this.isShutdown) {\n startBlockHeight = getStartBlockHeight();\n\n scaledBatchSize = this.blockDispatcher.smartBatchSize;\n\n if (scaledBatchSize === 0) {\n await waitForBatchSize(this.blockDispatcher.minimumHeapLimit);\n continue;\n }\n\n const latestHeight = this.nodeConfig.unfinalizedBlocks\n ? this.latestBestHeight\n : this.latestFinalizedHeight;\n\n if (\n this.blockDispatcher.freeSize < scaledBatchSize ||\n startBlockHeight > latestHeight\n ) {\n await delay(1);\n continue;\n }\n\n if (\n this.useDictionary &&\n startBlockHeight >= this.dictionaryService.startHeight\n ) {\n const queryEndBlock = startBlockHeight + DICTIONARY_MAX_QUERY_SIZE;\n const moduloBlocks = this.getModuloBlocks(\n startBlockHeight,\n queryEndBlock,\n );\n\n try {\n const dictionary =\n await this.dictionaryService.scopedDictionaryEntries(\n startBlockHeight,\n queryEndBlock,\n scaledBatchSize,\n );\n\n if (startBlockHeight !== getStartBlockHeight()) {\n logger.debug(\n `Queue was reset for new DS, discarding dictionary query result`,\n );\n continue;\n }\n\n if (\n dictionary &&\n this.dictionaryValidation(dictionary, startBlockHeight)\n ) {\n let { batchBlocks } = dictionary;\n\n batchBlocks = batchBlocks\n .concat(moduloBlocks)\n .sort((a, b) => a - b);\n if (batchBlocks.length === 0) {\n // There we're no blocks in this query range, we can set a new height we're up to\n await this.blockDispatcher.enqueueBlocks(\n [],\n Math.min(\n queryEndBlock - 1,\n dictionary._metadata.lastProcessedHeight,\n ),\n );\n } else {\n const maxBlockSize = Math.min(\n batchBlocks.length,\n this.blockDispatcher.freeSize,\n );\n const enqueuingBlocks = batchBlocks.slice(0, maxBlockSize);\n const cleanedBatchBlocks =\n this.filteredBlockBatch(enqueuingBlocks);\n await this.blockDispatcher.enqueueBlocks(\n cleanedBatchBlocks,\n this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n );\n }\n continue; // skip nextBlockRange() way\n }\n // else use this.nextBlockRange()\n } catch (e) {\n logger.debug(`Fetch dictionary stopped: ${e.message}`);\n this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n }\n }\n\n const endHeight = this.nextEndBlockHeight(\n startBlockHeight,\n scaledBatchSize,\n );\n\n const enqueuingBlocks =\n handlers.length && this.getModulos().length === handlers.length\n ? this.getEnqueuedModuloBlocks(startBlockHeight)\n : range(startBlockHeight, endHeight + 1);\n\n const cleanedBatchBlocks = this.filteredBlockBatch(enqueuingBlocks);\n await this.blockDispatcher.enqueueBlocks(\n cleanedBatchBlocks,\n this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n );\n }\n }\n\n private getLatestBufferHeight(\n cleanedBatchBlocks: number[],\n rawBatchBlocks: number[],\n ): number {\n return Math.max(...cleanedBatchBlocks, ...rawBatchBlocks);\n }\n private filteredBlockBatch(currentBatchBlocks: number[]): number[] {\n if (!this.bypassBlocks.length || !currentBatchBlocks) {\n return currentBatchBlocks;\n }\n\n const cleanedBatch = cleanedBatchBlocks(\n this.bypassBlocks,\n currentBatchBlocks,\n );\n\n const pollutedBlocks = this.bypassBlocks.filter(\n (b) => b < Math.max(...currentBatchBlocks),\n );\n if (pollutedBlocks.length) {\n logger.info(`Bypassing blocks: ${pollutedBlocks}`);\n }\n this.bypassBlocks = without(this.bypassBlocks, ...pollutedBlocks);\n return cleanedBatch;\n }\n\n private nextEndBlockHeight(\n startBlockHeight: number,\n scaledBatchSize: number,\n ): number {\n let endBlockHeight = startBlockHeight + scaledBatchSize - 1;\n\n if (endBlockHeight > this.latestFinalizedHeight) {\n if (this.nodeConfig.unfinalizedBlocks) {\n if (endBlockHeight >= this.latestBestHeight) {\n endBlockHeight = this.latestBestHeight;\n }\n } else {\n endBlockHeight = this.latestFinalizedHeight;\n }\n }\n return endBlockHeight;\n }\n\n private dictionaryValidation(\n dictionary: { _metadata: MetaData },\n startBlockHeight?: number,\n ): boolean {\n const validate = (): boolean => {\n if (dictionary !== undefined) {\n const { _metadata: metaData } = dictionary;\n\n if (\n metaData.genesisHash !== this.api.getGenesisHash() &&\n this.evmChainId !== this.api.getChainId().toString()\n ) {\n logger.error(\n 'The dictionary that you have specified does not match the chain you are indexing, it will be ignored. Please update your project manifest to reference the correct dictionary',\n );\n return false;\n }\n\n if (\n startBlockHeight !== undefined &&\n metaData.lastProcessedHeight < startBlockHeight\n ) {\n logger.warn(\n `Dictionary indexed block is behind current indexing block height`,\n );\n return false;\n }\n return true;\n }\n return false;\n };\n\n const valid = validate();\n\n this.dictionaryMetaValid = valid;\n this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n value: Number(this.useDictionary),\n });\n this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n\n return valid;\n }\n\n async resetForNewDs(blockHeight: number): Promise<void> {\n await this.syncDynamicDatascourcesFromMeta();\n this.dynamicDsService.deleteTempDsRecords(blockHeight);\n this.updateDictionary();\n this.blockDispatcher.flushQueue(blockHeight);\n }\n async resetForIncorrectBestBlock(blockHeight: number): Promise<void> {\n await this.syncDynamicDatascourcesFromMeta();\n this.updateDictionary();\n this.blockDispatcher.flushQueue(blockHeight);\n }\n}\n"]}
package/dist/indexer/indexer.manager.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"indexer.manager.js","sourceRoot":"","sources":["../../src/indexer/indexer.manager.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,4DAUgC;AAChC,gDAQ0B;AAU1B,+DAAkE;AAClE,oCAAwC;AACxC,iEAGgC;AAChC,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AAEvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IACzB,YACU,UAAsB,EACtB,UAAsB,EACtB,cAA8B,EAC9B,gBAAkC,EAClC,wBAAkD,EAClD,kBAAsC,EACX,cAA8B;QANzD,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;QAC9B,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,uBAAkB,GAAlB,kBAAkB,CAAoB;QACX,mBAAc,GAAd,cAAc,CAAgB;QAEjE,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACvC,CAAC;IAGK,AAAN,KAAK,CAAC,UAAU,CACd,YAAkC;QAElC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,YAAY,CAAC;QAC5C,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,kBAAkB,GAAG,IAAI,CAAC;QAE9B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAC7D,WAAW,CACZ,CAAC;QAEF,uCAAuC;QACvC,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QAEjD,kBAAkB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAEhE,iDAAiD;QACjD,IAAI,CAAC,kBAAkB,EAAE;YACvB,MAAM,IAAI,CAAC,cAAc,CACvB,YAAY,EACZ,WAAW;YACX,4DAA4D;YAC5D,KAAK,EAAE,EAAkB,EAAE,EAAE;gBAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAClD,EAAE,EACF,IAAI,CAAC,UAAU,CAAC,GAAG,EACnB,YAAY,CACb,CAAC;gBAEF,uCAAuC;gBACvC,EAAE,CAAC,MAAM,CACP,KAAK,EAAE,YAAoB,EAAE,IAA8B,EAAE,EAAE;oBAC7D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,uBAAuB,CAC/D;wBACE,YAAY;wBACZ,IAAI;wBACJ,UAAU,EAAE,WAAW;qBACxB,CACF,CAAC;oBACF,+FAA+F;oBAC/F,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACxB,gBAAgB,GAAG,IAAI,CAAC;gBAC1B,CAAC,EACD,yBAAyB,CAC1B,CAAC;gBAEF,OAAO,EAAE,CAAC;YACZ,CAAC,CACF,CAAC;SACH;QAED,OAAO;YACL,gBAAgB;YAChB,SAAS,EAAE,KAAK,CAAC,IAAI;YACrB,kBAAkB;SACnB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,KAAoB;QAEpB,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,OAAO,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;SACtE;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,iBAAiB,CAAC,EAAoB,EAAE,WAAmB;QACjE,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;YACd,MAAM,CAAC,KAAK,CACV,0IAA0I,WAAW;yFACpE,CAClF,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,EAAE,KAAK,EAAE,YAAY,EAAwB,EAC7C,WAA6B,EAC7B,KAAqD;;QAErD,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAExD,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE;YAC7B,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YAEpD,KAAK,MAAM,GAAG,IAAI,MAAA,EAAE,CAAC,IAAI,mCAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;aAChD;SACF;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,KAAoB,EACpB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACnE;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,EAAuB,EACvB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SAC/D;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,GAAgB,EAChB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACjE;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,IAAO,EACP,IAAuC,EACvC,EAAkB,EAClB,KAAsD;QAEtD,IAAI,EAAkB,CAAC;QACvB,IAAI,IAAA,6BAAW,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,QAAQ,GAAI,EAAE,CAAC,OAAO,CAAC,QAAkC,CAAC,MAAM,CACpE,CAAC,CAAC,EAAE,EAAE;;gBACJ,OAAA,CAAC,CAAC,IAAI,KAAK,IAAI;oBACf,aAAa,CAAC,IAAI,CAAC,CACjB,IAAW,EACX,CAAC,CAAC,MAAa,EACf,MAAA,EAAE,CAAC,OAAO,0CAAE,OAAO,CACpB,CAAA;aAAA,CACJ,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;gBACpB,OAAO;aACR;YACD,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAC/D,IAAI,EACJ,EAAE,CACH,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,EAAE,GAAG,EAAE,aAAF,EAAE,cAAF,EAAE,GAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,IAAI,CAAC,UAAU,CAAC,QAAQ;oBACtB,CAAC,CAAC,MAAM,IAAA,wBAAY,EAChB,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,EACvB,oBAAoB,EACpB,OAAO,CAAC,OAAO,CAChB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;oBAClC,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;aACzD;SACF;aAAM,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAC1C,EAAE,EACF,IAAI,EACJ,gBAAgB,CAAC,IAAI,CAAC,EACtB,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE;gBACnB,QAAQ,IAAI,EAAE;oBACZ,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,qCAAoB,CAAC,qBAAqB,CAC/C,IAAqB,EACrB,UAAU,CACX,CAAC;oBACJ,KAAK,qCAAmB,CAAC,IAAI;wBAC3B,OAAO,qCAAoB,CAAC,2BAA2B,CACrD,IAA2B,EAC3B,UAAU,CACX,CAAC;oBACJ,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,qCAAoB,CAAC,mBAAmB,CAC7C,IAAmB,EACnB,UAAU,CACX,CAAC;oBACJ;wBACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;iBAC/C;YACH,CAAC,CACF,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;gBACpB,OAAO;aACR;YAED,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAC/D,IAAI,EACJ,EAAE,CACH,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,EAAE,GAAG,EAAE,aAAF,EAAE,cAAF,EAAE,GAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,MAAM,IAAI,CAAC,2BAA2B,CAAC,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;aACrE;SACF;IACH,CAAC;IAEO,sBAAsB,CAC5B,EAA8C,EAC9C,IAAuC,EACvC,gBAAqC,EACrC,UAGY;QAEZ,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAE1D,OAAO,EAAE,CAAC,OAAO,CAAC,QAAQ;aACvB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACzD,IAAI,gBAAgB,CAAC,SAAS,CAAC,EAAE;gBAC/B,SAAS,CAAC,UAAU,CAAC;gBAErB,OAAO,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;aAC/C;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,IAAA,0DAAmC,EACnD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CACvC,CAAC;YAEF,IAAI;gBACF,OAAO,SAAS,CAAC,eAAe,CAAC;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,KAAK,EAAE,IAAI;oBACX,EAAE;iBACH,CAAC,CAAC;aACJ;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oCAAoC,CAAC,CAAC;gBACtD,MAAM,CAAC,CAAC;aACT;QACH,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,EAA8C,EAC9C,EAAkB,EAClB,OAA2B,EAC3B,IAAuC;QAEvC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAE3D,MAAM,SAAS,GAAG,IAAA,0DAAmC,EACnD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CACvC,CAAC;QAEF,MAAM,eAAe,GAAG,MAAM,SAAS;aACpC,WAAW,CAAC;YACX,KAAK,EAAE,IAAI;YACX,EAAE;YACF,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG;YACxB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM;SACP,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,6CAA6C,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;QACV,CAAC,CAAC,CAAC;QAEL,+FAA+F;QAC/F,+FAA+F;QAC/F,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE;YACnC,MAAM,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;SAChD;IACH,CAAC;CACF,CAAA;AAvRO;IADL,IAAA,oBAAQ,EAAC,oBAAY,CAAC,IAAI,CAAC,QAAQ,CAAC;;;;gDAyDpC;AAtEU,cAAc;IAD1B,IAAA,mBAAU,GAAE;IASR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;qCANN,sBAAU;QACV,sBAAU;QACN,gCAAc;QACZ,qCAAgB;QACR,oDAAwB;QAC9B,yCAAkB;QACK,gCAAc;GARxD,cAAc,CAqS1B;AArSY,wCAAc;AA6S3B,MAAM,gBAAgB,GAAG;IACvB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,wCAAsB;CACnD,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,qBAAqB;IACvE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,mBAAmB;IACrE,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,qCAAoB,CAAC,2BAA2B;CAC7E,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,IAAmB,EAAE,EAAE,CAAC,IAAI;IAChE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,CAAC,GAAgB,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;IACzE,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,GAAgB,EAAE,EAAE,CAC/C,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC;CACjC,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n isBlockHandlerProcessor,\n isCallHandlerProcessor,\n isEventHandlerProcessor,\n isCustomDs,\n isRuntimeDs,\n SubqlEthereumCustomDataSource,\n SubqlCustomHandler,\n EthereumHandlerKind,\n EthereumRuntimeHandlerInputMap,\n} from '@subql/common-ethereum';\nimport {\n NodeConfig,\n getLogger,\n profiler,\n profilerWrap,\n IndexerSandbox,\n ProcessBlockResponse,\n ApiService,\n} from '@subql/node-core';\nimport {\n EthereumTransaction,\n EthereumLog,\n SubqlRuntimeHandler,\n EthereumBlockWrapper,\n EthereumBlock,\n} from '@subql/types-ethereum';\nimport { SubqlProjectDs } from '../configure/SubqueryProject';\nimport { EthereumApi } from '../ethereum';\nimport { EthereumBlockWrapped } from '../ethereum/block.ethereum';\nimport { yargsOptions } from '../yargs';\nimport {\n asSecondLayerHandlerProcessor_1_0_0,\n DsProcessorService,\n} from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\nconst logger = getLogger('indexer');\n\n@Injectable()\nexport class IndexerManager {\n constructor(\n private apiService: ApiService,\n private nodeConfig: NodeConfig,\n private sandboxService: SandboxService,\n private dynamicDsService: DynamicDsService,\n private unfinalizedBlocksService: UnfinalizedBlocksService,\n private dsProcessorService: DsProcessorService,\n @Inject('IProjectService') private projectService: ProjectService,\n ) {\n logger.info('indexer manager start');\n }\n\n @profiler(yargsOptions.argv.profiler)\n async indexBlock(\n blockContent: EthereumBlockWrapper,\n ): Promise<ProcessBlockResponse> {\n const { block, blockHeight } = blockContent;\n let dynamicDsCreated = false;\n let reindexBlockHeight = null;\n\n const datasources = await this.projectService.getAllDataSources(\n blockHeight,\n );\n\n // Check that we have valid datasources\n this.assertDataSources(datasources, blockHeight);\n\n reindexBlockHeight = await this.processUnfinalizedBlocks(block);\n\n // Only index block if we're not going to reindex\n if (!reindexBlockHeight) {\n await this.indexBlockData(\n blockContent,\n datasources,\n // eslint-disable-next-line @typescript-eslint/require-await\n async (ds: SubqlProjectDs) => {\n const vm = this.sandboxService.getDsProcessorWrapper(\n ds,\n this.apiService.api,\n blockContent,\n );\n\n // Inject function to create ds into vm\n vm.freeze(\n async (templateName: string, args?: Record<string, unknown>) => {\n const newDs = await this.dynamicDsService.createDynamicDatasource(\n {\n templateName,\n args,\n startBlock: blockHeight,\n },\n );\n // Push the newly created dynamic ds to be processed this block on any future extrinsics/events\n datasources.push(newDs);\n dynamicDsCreated = true;\n },\n 'createDynamicDatasource',\n );\n\n return vm;\n },\n );\n }\n\n return {\n dynamicDsCreated,\n blockHash: block.hash,\n reindexBlockHeight,\n };\n }\n\n async start(): Promise<void> {\n await this.projectService.init();\n logger.info('indexer manager started');\n }\n\n private async processUnfinalizedBlocks(\n block: EthereumBlock,\n ): Promise<number | null> {\n if (this.nodeConfig.unfinalizedBlocks) {\n return this.unfinalizedBlocksService.processUnfinalizedBlocks(block);\n }\n return null;\n }\n\n private assertDataSources(ds: SubqlProjectDs[], blockHeight: number) {\n if (!ds.length) {\n logger.error(\n `Your start block is greater than the current indexed block height in your database. Either change your startBlock (project.yaml) to <= ${blockHeight}\n or delete your database and start again from the currently specified startBlock`,\n );\n process.exit(1);\n }\n }\n\n private async indexBlockData(\n { block, transactions }: EthereumBlockWrapper,\n dataSources: SubqlProjectDs[],\n getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n ): Promise<void> {\n await this.indexBlockContent(block, dataSources, getVM);\n\n for (const tx of transactions) {\n await this.indexTransaction(tx, dataSources, getVM);\n\n for (const log of tx.logs ?? []) {\n await this.indexEvent(log, dataSources, getVM);\n }\n }\n }\n\n private async indexBlockContent(\n block: EthereumBlock,\n dataSources: SubqlProjectDs[],\n getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n ): Promise<void> {\n for (const ds of dataSources) {\n await this.indexData(EthereumHandlerKind.Block, block, ds, getVM);\n }\n }\n\n private async indexTransaction(\n tx: EthereumTransaction,\n dataSources: SubqlProjectDs[],\n getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n ): Promise<void> {\n for (const ds of dataSources) {\n await this.indexData(EthereumHandlerKind.Call, tx, ds, getVM);\n }\n }\n\n private async indexEvent(\n log: EthereumLog,\n dataSources: SubqlProjectDs[],\n getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n ): Promise<void> {\n for (const ds of dataSources) {\n await this.indexData(EthereumHandlerKind.Event, log, ds, getVM);\n }\n }\n\n private async indexData<K extends EthereumHandlerKind>(\n kind: K,\n data: EthereumRuntimeHandlerInputMap[K],\n ds: SubqlProjectDs,\n getVM: (ds: SubqlProjectDs) => Promise<IndexerSandbox>,\n ): Promise<void> {\n let vm: IndexerSandbox;\n if (isRuntimeDs(ds)) {\n const handlers = (ds.mapping.handlers as SubqlRuntimeHandler[]).filter(\n (h) =>\n h.kind === kind &&\n FilterTypeMap[kind](\n data as any,\n h.filter as any,\n ds.options?.address,\n ),\n );\n\n if (!handlers.length) {\n return;\n }\n const parsedData = await DataAbiParser[kind](this.apiService.api)(\n data,\n ds,\n );\n\n for (const handler of handlers) {\n vm = vm ?? (await getVM(ds));\n this.nodeConfig.profiler\n ? await profilerWrap(\n vm.securedExec.bind(vm),\n 'handlerPerformance',\n handler.handler,\n )(handler.handler, [parsedData])\n : await vm.securedExec(handler.handler, [parsedData]);\n }\n } else if (isCustomDs(ds)) {\n const handlers = this.filterCustomDsHandlers<K>(\n ds,\n data,\n ProcessorTypeMap[kind],\n (data, baseFilter) => {\n switch (kind) {\n case EthereumHandlerKind.Block:\n return EthereumBlockWrapped.filterBlocksProcessor(\n data as EthereumBlock,\n baseFilter,\n );\n case EthereumHandlerKind.Call:\n return EthereumBlockWrapped.filterTransactionsProcessor(\n data as EthereumTransaction,\n baseFilter,\n );\n case EthereumHandlerKind.Event:\n return EthereumBlockWrapped.filterLogsProcessor(\n data as EthereumLog,\n baseFilter,\n );\n default:\n throw new Error('Unsupported handler kind');\n }\n },\n );\n\n if (!handlers.length) {\n return;\n }\n\n const parsedData = await DataAbiParser[kind](this.apiService.api)(\n data,\n ds,\n );\n\n for (const handler of handlers) {\n vm = vm ?? (await getVM(ds));\n await this.transformAndExecuteCustomDs(ds, vm, handler, parsedData);\n }\n }\n }\n\n private filterCustomDsHandlers<K extends EthereumHandlerKind>(\n ds: SubqlEthereumCustomDataSource<string, any>,\n data: EthereumRuntimeHandlerInputMap[K],\n baseHandlerCheck: ProcessorTypeMap[K],\n baseFilter: (\n data: EthereumRuntimeHandlerInputMap[K],\n baseFilter: any,\n ) => boolean,\n ): SubqlCustomHandler[] {\n const plugin = this.dsProcessorService.getDsProcessor(ds);\n\n return ds.mapping.handlers\n .filter((handler) => {\n const processor = plugin.handlerProcessors[handler.kind];\n if (baseHandlerCheck(processor)) {\n processor.baseFilter;\n\n return baseFilter(data, processor.baseFilter);\n }\n return false;\n })\n .filter((handler) => {\n const processor = asSecondLayerHandlerProcessor_1_0_0(\n plugin.handlerProcessors[handler.kind],\n );\n\n try {\n return processor.filterProcessor({\n filter: handler.filter,\n input: data,\n ds,\n });\n } catch (e) {\n logger.error(e, 'Failed to run ds processer filter.');\n throw e;\n }\n });\n }\n\n private async transformAndExecuteCustomDs<K extends EthereumHandlerKind>(\n ds: SubqlEthereumCustomDataSource<string, any>,\n vm: IndexerSandbox,\n handler: SubqlCustomHandler,\n data: EthereumRuntimeHandlerInputMap[K],\n ): Promise<void> {\n const plugin = this.dsProcessorService.getDsProcessor(ds);\n const assets = await this.dsProcessorService.getAssets(ds);\n\n const processor = asSecondLayerHandlerProcessor_1_0_0(\n plugin.handlerProcessors[handler.kind],\n );\n\n const transformedData = await processor\n .transformer({\n input: data,\n ds,\n api: this.apiService.api,\n filter: handler.filter,\n assets,\n })\n .catch((e) => {\n logger.error(e, 'Failed to transform data with ds processor.');\n throw e;\n });\n\n // We can not run this in parallel. the transformed data items may be dependent on one another.\n // An example of this is with Acala EVM packing multiple EVM logs into a single Substrate event\n for (const _data of transformedData) {\n await vm.securedExec(handler.handler, [_data]);\n }\n }\n}\n\ntype ProcessorTypeMap = {\n [EthereumHandlerKind.Block]: typeof isBlockHandlerProcessor;\n [EthereumHandlerKind.Event]: typeof isEventHandlerProcessor;\n [EthereumHandlerKind.Call]: typeof isCallHandlerProcessor;\n};\n\nconst ProcessorTypeMap = {\n [EthereumHandlerKind.Block]: isBlockHandlerProcessor,\n [EthereumHandlerKind.Event]: isEventHandlerProcessor,\n [EthereumHandlerKind.Call]: isCallHandlerProcessor,\n};\n\nconst FilterTypeMap = {\n [EthereumHandlerKind.Block]: EthereumBlockWrapped.filterBlocksProcessor,\n [EthereumHandlerKind.Event]: EthereumBlockWrapped.filterLogsProcessor,\n [EthereumHandlerKind.Call]: EthereumBlockWrapped.filterTransactionsProcessor,\n};\n\nconst DataAbiParser = {\n [EthereumHandlerKind.Block]: () => (data: EthereumBlock) => data,\n [EthereumHandlerKind.Event]: (api: EthereumApi) => api.parseLog.bind(api),\n [EthereumHandlerKind.Call]: (api: EthereumApi) =>\n api.parseTransaction.bind(api),\n};\n"]}
1
+ {"version":3,"file":"indexer.manager.js","sourceRoot":"","sources":["../../src/indexer/indexer.manager.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,4DAUgC;AAChC,gDAQ0B;AAU1B,+DAAkE;AAClE,oCAAwC;AACxC,iEAGgC;AAChC,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AAEvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAGpC,IAAa,cAAc,GAA3B,MAAa,cAAc;IACzB,YACU,UAAsB,EACtB,UAAsB,EACtB,cAA8B,EAC9B,gBAAkC,EAClC,wBAAkD,EAClD,kBAAsC,EACX,cAA8B;QANzD,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;QAC9B,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,uBAAkB,GAAlB,kBAAkB,CAAoB;QACX,mBAAc,GAAd,cAAc,CAAgB;QAEjE,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACvC,CAAC;IAGD,KAAK,CAAC,UAAU,CACd,YAAkC;QAElC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,YAAY,CAAC;QAC5C,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,kBAAkB,GAAG,IAAI,CAAC;QAE9B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAC7D,WAAW,CACZ,CAAC;QAEF,uCAAuC;QACvC,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACjD,kBAAkB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAEhE,iDAAiD;QACjD,IAAI,CAAC,kBAAkB,EAAE;YACvB,MAAM,IAAI,CAAC,cAAc,CACvB,YAAY,EACZ,WAAW;YACX,4DAA4D;YAC5D,KAAK,EAAE,EAAkB,EAAE,EAAE;gBAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAClD,EAAE,EACF,IAAI,CAAC,UAAU,CAAC,GAAG,EACnB,YAAY,CACb,CAAC;gBAEF,uCAAuC;gBACvC,EAAE,CAAC,MAAM,CACP,KAAK,EAAE,YAAoB,EAAE,IAA8B,EAAE,EAAE;oBAC7D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,uBAAuB,CAC/D;wBACE,YAAY;wBACZ,IAAI;wBACJ,UAAU,EAAE,WAAW;qBACxB,CACF,CAAC;oBACF,+FAA+F;oBAC/F,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACxB,gBAAgB,GAAG,IAAI,CAAC;gBAC1B,CAAC,EACD,yBAAyB,CAC1B,CAAC;gBAEF,OAAO,EAAE,CAAC;YACZ,CAAC,CACF,CAAC;SACH;QAED,OAAO;YACL,gBAAgB;YAChB,SAAS,EAAE,KAAK,CAAC,IAAI;YACrB,kBAAkB;SACnB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,KAAoB;QAEpB,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,OAAO,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;SACtE;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,iBAAiB,CAAC,EAAoB,EAAE,WAAmB;QACjE,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;YACd,MAAM,CAAC,KAAK,CACV,0IAA0I,WAAW;yFACpE,CAClF,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,EAAE,KAAK,EAAE,YAAY,EAAwB,EAC7C,WAA6B,EAC7B,KAAqD;;QAErD,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAExD,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE;YAC7B,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YAEpD,KAAK,MAAM,GAAG,IAAI,MAAA,EAAE,CAAC,IAAI,mCAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;aAChD;SACF;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,KAAoB,EACpB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACnE;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,EAAuB,EACvB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SAC/D;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,GAAgB,EAChB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACjE;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,IAAO,EACP,IAAuC,EACvC,EAAkB,EAClB,KAAsD;QAEtD,IAAI,EAAkB,CAAC;QACvB,IAAI,IAAA,6BAAW,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,QAAQ,GAAI,EAAE,CAAC,OAAO,CAAC,QAAkC,CAAC,MAAM,CACpE,CAAC,CAAC,EAAE,EAAE;;gBACJ,OAAA,CAAC,CAAC,IAAI,KAAK,IAAI;oBACf,aAAa,CAAC,IAAI,CAAC,CACjB,IAAW,EACX,CAAC,CAAC,MAAa,EACf,MAAA,EAAE,CAAC,OAAO,0CAAE,OAAO,CACpB,CAAA;aAAA,CACJ,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;gBACpB,OAAO;aACR;YACD,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAC/D,IAAI,EACJ,EAAE,CACH,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,EAAE,GAAG,EAAE,aAAF,EAAE,cAAF,EAAE,GAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,IAAI,CAAC,UAAU,CAAC,QAAQ;oBACtB,CAAC,CAAC,MAAM,IAAA,wBAAY,EAChB,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,EACvB,oBAAoB,EACpB,OAAO,CAAC,OAAO,CAChB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;oBAClC,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;aACzD;SACF;aAAM,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAC1C,EAAE,EACF,IAAI,EACJ,gBAAgB,CAAC,IAAI,CAAC,EACtB,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE;gBACnB,QAAQ,IAAI,EAAE;oBACZ,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,qCAAoB,CAAC,qBAAqB,CAC/C,IAAqB,EACrB,UAAU,CACX,CAAC;oBACJ,KAAK,qCAAmB,CAAC,IAAI;wBAC3B,OAAO,qCAAoB,CAAC,2BAA2B,CACrD,IAA2B,EAC3B,UAAU,CACX,CAAC;oBACJ,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,qCAAoB,CAAC,mBAAmB,CAC7C,IAAmB,EACnB,UAAU,CACX,CAAC;oBACJ;wBACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;iBAC/C;YACH,CAAC,CACF,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;gBACpB,OAAO;aACR;YAED,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAC/D,IAAI,EACJ,EAAE,CACH,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,EAAE,GAAG,EAAE,aAAF,EAAE,cAAF,EAAE,GAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,MAAM,IAAI,CAAC,2BAA2B,CAAC,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;aACrE;SACF;IACH,CAAC;IAEO,sBAAsB,CAC5B,EAA8C,EAC9C,IAAuC,EACvC,gBAAqC,EACrC,UAGY;QAEZ,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAE1D,OAAO,EAAE,CAAC,OAAO,CAAC,QAAQ;aACvB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACzD,IAAI,gBAAgB,CAAC,SAAS,CAAC,EAAE;gBAC/B,SAAS,CAAC,UAAU,CAAC;gBAErB,OAAO,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;aAC/C;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,IAAA,0DAAmC,EACnD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CACvC,CAAC;YAEF,IAAI;gBACF,OAAO,SAAS,CAAC,eAAe,CAAC;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,KAAK,EAAE,IAAI;oBACX,EAAE;iBACH,CAAC,CAAC;aACJ;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oCAAoC,CAAC,CAAC;gBACtD,MAAM,CAAC,CAAC;aACT;QACH,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,EAA8C,EAC9C,EAAkB,EAClB,OAA2B,EAC3B,IAAuC;QAEvC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAE3D,MAAM,SAAS,GAAG,IAAA,0DAAmC,EACnD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CACvC,CAAC;QAEF,MAAM,eAAe,GAAG,MAAM,SAAS;aACpC,WAAW,CAAC;YACX,KAAK,EAAE,IAAI;YACX,EAAE;YACF,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG;YACxB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM;SACP,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,6CAA6C,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;QACV,CAAC,CAAC,CAAC;QAEL,+FAA+F;QAC/F,+FAA+F;QAC/F,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE;YACnC,MAAM,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;SAChD;IACH,CAAC;CACF,CAAA;AAtRC;IADC,IAAA,oBAAQ,EAAC,oBAAY,CAAC,IAAI,CAAC,QAAQ,CAAC;;;;gDAwDpC;AArEU,cAAc;IAD1B,IAAA,mBAAU,GAAE;IASR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;qCANN,sBAAU;QACV,sBAAU;QACN,gCAAc;QACZ,qCAAgB;QACR,oDAAwB;QAC9B,yCAAkB;QACK,gCAAc;GARxD,cAAc,CAoS1B;AApSY,wCAAc;AA4S3B,MAAM,gBAAgB,GAAG;IACvB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,wCAAsB;CACnD,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,qBAAqB;IACvE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,mBAAmB;IACrE,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,qCAAoB,CAAC,2BAA2B;CAC7E,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,IAAmB,EAAE,EAAE,CAAC,IAAI;IAChE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,CAAC,GAAgB,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;IACzE,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,GAAgB,EAAE,EAAE,CAC/C,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC;CACjC,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n isBlockHandlerProcessor,\n isCallHandlerProcessor,\n isEventHandlerProcessor,\n isCustomDs,\n isRuntimeDs,\n SubqlEthereumCustomDataSource,\n SubqlCustomHandler,\n EthereumHandlerKind,\n EthereumRuntimeHandlerInputMap,\n} from '@subql/common-ethereum';\nimport {\n NodeConfig,\n getLogger,\n profiler,\n profilerWrap,\n IndexerSandbox,\n ProcessBlockResponse,\n ApiService,\n} from '@subql/node-core';\nimport {\n EthereumTransaction,\n EthereumLog,\n SubqlRuntimeHandler,\n EthereumBlockWrapper,\n EthereumBlock,\n} from '@subql/types-ethereum';\nimport { SubqlProjectDs } from '../configure/SubqueryProject';\nimport { EthereumApi } from '../ethereum';\nimport { EthereumBlockWrapped } from '../ethereum/block.ethereum';\nimport { yargsOptions } from '../yargs';\nimport {\n asSecondLayerHandlerProcessor_1_0_0,\n DsProcessorService,\n} from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\nconst logger = getLogger('indexer');\n\n@Injectable()\nexport class IndexerManager {\n constructor(\n private apiService: ApiService,\n private nodeConfig: NodeConfig,\n private sandboxService: SandboxService,\n private dynamicDsService: DynamicDsService,\n private unfinalizedBlocksService: UnfinalizedBlocksService,\n private dsProcessorService: DsProcessorService,\n @Inject('IProjectService') private projectService: ProjectService,\n ) {\n logger.info('indexer manager start');\n }\n\n @profiler(yargsOptions.argv.profiler)\n async indexBlock(\n blockContent: EthereumBlockWrapper,\n ): Promise<ProcessBlockResponse> {\n const { block, blockHeight } = blockContent;\n let dynamicDsCreated = false;\n let reindexBlockHeight = null;\n\n const datasources = await this.projectService.getAllDataSources(\n blockHeight,\n );\n\n // Check that we have valid datasources\n this.assertDataSources(datasources, blockHeight);\n reindexBlockHeight = await this.processUnfinalizedBlocks(block);\n\n // Only index block if we're not going to reindex\n if (!reindexBlockHeight) {\n await this.indexBlockData(\n blockContent,\n datasources,\n // eslint-disable-next-line @typescript-eslint/require-await\n async (ds: SubqlProjectDs) => {\n const vm = this.sandboxService.getDsProcessorWrapper(\n ds,\n this.apiService.api,\n blockContent,\n );\n\n // Inject function to create ds into vm\n vm.freeze(\n async (templateName: string, args?: Record<string, unknown>) => {\n const newDs = await this.dynamicDsService.createDynamicDatasource(\n {\n templateName,\n args,\n startBlock: blockHeight,\n },\n );\n // Push the newly created dynamic ds to be processed this block on any future extrinsics/events\n datasources.push(newDs);\n dynamicDsCreated = true;\n },\n 'createDynamicDatasource',\n );\n\n return vm;\n },\n );\n }\n\n return {\n dynamicDsCreated,\n blockHash: block.hash,\n reindexBlockHeight,\n };\n }\n\n async start(): Promise<void> {\n await this.projectService.init();\n logger.info('indexer manager started');\n }\n\n private async processUnfinalizedBlocks(\n block: EthereumBlock,\n ): Promise<number | null> {\n if (this.nodeConfig.unfinalizedBlocks) {\n return this.unfinalizedBlocksService.processUnfinalizedBlocks(block);\n }\n return null;\n }\n\n private assertDataSources(ds: SubqlProjectDs[], blockHeight: number) {\n if (!ds.length) {\n logger.error(\n `Your start block is greater than the current indexed block height in your database. Either change your startBlock (project.yaml) to <= ${blockHeight}\n or delete your database and start again from the currently specified startBlock`,\n );\n process.exit(1);\n }\n }\n\n private async indexBlockData(\n { block, transactions }: EthereumBlockWrapper,\n dataSources: SubqlProjectDs[],\n getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n ): Promise<void> {\n await this.indexBlockContent(block, dataSources, getVM);\n\n for (const tx of transactions) {\n await this.indexTransaction(tx, dataSources, getVM);\n\n for (const log of tx.logs ?? []) {\n await this.indexEvent(log, dataSources, getVM);\n }\n }\n }\n\n private async indexBlockContent(\n block: EthereumBlock,\n dataSources: SubqlProjectDs[],\n getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n ): Promise<void> {\n for (const ds of dataSources) {\n await this.indexData(EthereumHandlerKind.Block, block, ds, getVM);\n }\n }\n\n private async indexTransaction(\n tx: EthereumTransaction,\n dataSources: SubqlProjectDs[],\n getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n ): Promise<void> {\n for (const ds of dataSources) {\n await this.indexData(EthereumHandlerKind.Call, tx, ds, getVM);\n }\n }\n\n private async indexEvent(\n log: EthereumLog,\n dataSources: SubqlProjectDs[],\n getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n ): Promise<void> {\n for (const ds of dataSources) {\n await this.indexData(EthereumHandlerKind.Event, log, ds, getVM);\n }\n }\n\n private async indexData<K extends EthereumHandlerKind>(\n kind: K,\n data: EthereumRuntimeHandlerInputMap[K],\n ds: SubqlProjectDs,\n getVM: (ds: SubqlProjectDs) => Promise<IndexerSandbox>,\n ): Promise<void> {\n let vm: IndexerSandbox;\n if (isRuntimeDs(ds)) {\n const handlers = (ds.mapping.handlers as SubqlRuntimeHandler[]).filter(\n (h) =>\n h.kind === kind &&\n FilterTypeMap[kind](\n data as any,\n h.filter as any,\n ds.options?.address,\n ),\n );\n\n if (!handlers.length) {\n return;\n }\n const parsedData = await DataAbiParser[kind](this.apiService.api)(\n data,\n ds,\n );\n\n for (const handler of handlers) {\n vm = vm ?? (await getVM(ds));\n this.nodeConfig.profiler\n ? await profilerWrap(\n vm.securedExec.bind(vm),\n 'handlerPerformance',\n handler.handler,\n )(handler.handler, [parsedData])\n : await vm.securedExec(handler.handler, [parsedData]);\n }\n } else if (isCustomDs(ds)) {\n const handlers = this.filterCustomDsHandlers<K>(\n ds,\n data,\n ProcessorTypeMap[kind],\n (data, baseFilter) => {\n switch (kind) {\n case EthereumHandlerKind.Block:\n return EthereumBlockWrapped.filterBlocksProcessor(\n data as EthereumBlock,\n baseFilter,\n );\n case EthereumHandlerKind.Call:\n return EthereumBlockWrapped.filterTransactionsProcessor(\n data as EthereumTransaction,\n baseFilter,\n );\n case EthereumHandlerKind.Event:\n return EthereumBlockWrapped.filterLogsProcessor(\n data as EthereumLog,\n baseFilter,\n );\n default:\n throw new Error('Unsupported handler kind');\n }\n },\n );\n\n if (!handlers.length) {\n return;\n }\n\n const parsedData = await DataAbiParser[kind](this.apiService.api)(\n data,\n ds,\n );\n\n for (const handler of handlers) {\n vm = vm ?? (await getVM(ds));\n await this.transformAndExecuteCustomDs(ds, vm, handler, parsedData);\n }\n }\n }\n\n private filterCustomDsHandlers<K extends EthereumHandlerKind>(\n ds: SubqlEthereumCustomDataSource<string, any>,\n data: EthereumRuntimeHandlerInputMap[K],\n baseHandlerCheck: ProcessorTypeMap[K],\n baseFilter: (\n data: EthereumRuntimeHandlerInputMap[K],\n baseFilter: any,\n ) => boolean,\n ): SubqlCustomHandler[] {\n const plugin = this.dsProcessorService.getDsProcessor(ds);\n\n return ds.mapping.handlers\n .filter((handler) => {\n const processor = plugin.handlerProcessors[handler.kind];\n if (baseHandlerCheck(processor)) {\n processor.baseFilter;\n\n return baseFilter(data, processor.baseFilter);\n }\n return false;\n })\n .filter((handler) => {\n const processor = asSecondLayerHandlerProcessor_1_0_0(\n plugin.handlerProcessors[handler.kind],\n );\n\n try {\n return processor.filterProcessor({\n filter: handler.filter,\n input: data,\n ds,\n });\n } catch (e) {\n logger.error(e, 'Failed to run ds processer filter.');\n throw e;\n }\n });\n }\n\n private async transformAndExecuteCustomDs<K extends EthereumHandlerKind>(\n ds: SubqlEthereumCustomDataSource<string, any>,\n vm: IndexerSandbox,\n handler: SubqlCustomHandler,\n data: EthereumRuntimeHandlerInputMap[K],\n ): Promise<void> {\n const plugin = this.dsProcessorService.getDsProcessor(ds);\n const assets = await this.dsProcessorService.getAssets(ds);\n\n const processor = asSecondLayerHandlerProcessor_1_0_0(\n plugin.handlerProcessors[handler.kind],\n );\n\n const transformedData = await processor\n .transformer({\n input: data,\n ds,\n api: this.apiService.api,\n filter: handler.filter,\n assets,\n })\n .catch((e) => {\n logger.error(e, 'Failed to transform data with ds processor.');\n throw e;\n });\n\n // We can not run this in parallel. the transformed data items may be dependent on one another.\n // An example of this is with Acala EVM packing multiple EVM logs into a single Substrate event\n for (const _data of transformedData) {\n await vm.securedExec(handler.handler, [_data]);\n }\n }\n}\n\ntype ProcessorTypeMap = {\n [EthereumHandlerKind.Block]: typeof isBlockHandlerProcessor;\n [EthereumHandlerKind.Event]: typeof isEventHandlerProcessor;\n [EthereumHandlerKind.Call]: typeof isCallHandlerProcessor;\n};\n\nconst ProcessorTypeMap = {\n [EthereumHandlerKind.Block]: isBlockHandlerProcessor,\n [EthereumHandlerKind.Event]: isEventHandlerProcessor,\n [EthereumHandlerKind.Call]: isCallHandlerProcessor,\n};\n\nconst FilterTypeMap = {\n [EthereumHandlerKind.Block]: EthereumBlockWrapped.filterBlocksProcessor,\n [EthereumHandlerKind.Event]: EthereumBlockWrapped.filterLogsProcessor,\n [EthereumHandlerKind.Call]: EthereumBlockWrapped.filterTransactionsProcessor,\n};\n\nconst DataAbiParser = {\n [EthereumHandlerKind.Block]: () => (data: EthereumBlock) => data,\n [EthereumHandlerKind.Event]: (api: EthereumApi) => api.parseLog.bind(api),\n [EthereumHandlerKind.Call]: (api: EthereumApi) =>\n api.parseTransaction.bind(api),\n};\n"]}
package/dist/indexer/indexer.module.js CHANGED
@@ -30,14 +30,15 @@ IndexerModule = __decorate([
30
30
  indexer_manager_1.IndexerManager,
31
31
  node_core_1.StoreCacheService,
32
32
  node_core_1.StoreService,
33
+ node_core_1.ConnectionPoolService,
33
34
  {
34
35
  provide: node_core_1.ApiService,
35
- useFactory: async (project, eventEmitter) => {
36
- const apiService = new ethereum_1.EthereumApiService(project, eventEmitter);
36
+ useFactory: async (project, connectionPoolService, eventEmitter) => {
37
+ const apiService = new ethereum_1.EthereumApiService(project, connectionPoolService, eventEmitter);
37
38
  await apiService.init();
38
39
  return apiService;
39
40
  },
40
- inject: ['ISubqueryProject', event_emitter_1.EventEmitter2],
41
+ inject: ['ISubqueryProject', node_core_1.ConnectionPoolService, event_emitter_1.EventEmitter2],
41
42
  },
42
43
  sandbox_service_1.SandboxService,
43
44
  ds_processor_service_1.DsProcessorService,
@@ -57,7 +58,6 @@ IndexerModule = __decorate([
57
58
  useClass: project_service_1.ProjectService,
58
59
  },
59
60
  worker_service_1.WorkerService,
60
- unfinalizedBlocks_service_1.UnfinalizedBlocksService,
61
61
  {
62
62
  provide: unfinalizedBlocks_service_1.UnfinalizedBlocksService,
63
63
  useFactory: () => {
package/dist/indexer/indexer.module.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"indexer.module.js","sourceRoot":"","sources":["../../src/indexer/indexer.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,mDAA8C;AAC9C,2CAAwC;AACxC,yDAAsD;AACtD,gDAO0B;AAE1B,0CAAiD;AACjD,iEAA4D;AAC5D,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AACvE,4DAAwD;AACxD,gGAA2F;AAkDpF,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IAhDzB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,gCAAc;YACd,6BAAiB;YACjB,wBAAY;YACZ;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EACf,OAAwB,EACxB,YAA2B,EAC3B,EAAE;oBACF,MAAM,UAAU,GAAG,IAAI,6BAAkB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;oBACjE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,6BAAa,CAAC;aAC5C;YACD,gCAAc;YACd,yCAAkB;YAClB;gBACE,OAAO,EAAE,qCAAgB;gBACzB,UAAU,EAAE,GAAG,EAAE;oBACf,IAAI,6BAAY,EAAE;wBAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;qBACjD;oBACD,OAAO,IAAI,kCAAsB,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC;gBAC1D,CAAC;aACF;YACD,sBAAU;YACV,sBAAU;YACV;gBACE,OAAO,EAAE,iBAAiB;gBAC1B,QAAQ,EAAE,gCAAc;aACzB;YACD,8BAAa;YACb,oDAAwB;YACxB;gBACE,OAAO,EAAE,oDAAwB;gBACjC,UAAU,EAAE,GAAG,EAAE;oBACf,IAAI,6BAAY,EAAE;wBAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;qBACjD;oBACD,OAAO,IAAI,iEAA8B,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC;gBAClE,CAAC;aACF;SACF;QACD,OAAO,EAAE,CAAC,wBAAY,CAAC;KACxB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { isMainThread } from 'worker_threads';\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n ApiService,\n StoreService,\n PoiService,\n MmrService,\n StoreCacheService,\n WorkerDynamicDsService,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\nimport { WorkerService } from './worker/worker.service';\nimport { WorkerUnfinalizedBlocksService } from './worker/worker.unfinalizedBlocks.service';\n\n@Module({\n providers: [\n IndexerManager,\n StoreCacheService,\n StoreService,\n {\n provide: ApiService,\n useFactory: async (\n project: SubqueryProject,\n eventEmitter: EventEmitter2,\n ) => {\n const apiService = new EthereumApiService(project, eventEmitter);\n await apiService.init();\n return apiService;\n },\n inject: ['ISubqueryProject', EventEmitter2],\n },\n SandboxService,\n DsProcessorService,\n {\n provide: DynamicDsService,\n useFactory: () => {\n if (isMainThread) {\n throw new Error('Expected to be worker thread');\n }\n return new WorkerDynamicDsService((global as any).host);\n },\n },\n PoiService,\n MmrService,\n {\n provide: 'IProjectService',\n useClass: ProjectService,\n },\n WorkerService,\n UnfinalizedBlocksService,\n {\n provide: UnfinalizedBlocksService,\n useFactory: () => {\n if (isMainThread) {\n throw new Error('Expected to be worker thread');\n }\n return new WorkerUnfinalizedBlocksService((global as any).host);\n },\n },\n ],\n exports: [StoreService],\n})\nexport class IndexerModule {}\n"]}
1
+ {"version":3,"file":"indexer.module.js","sourceRoot":"","sources":["../../src/indexer/indexer.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,mDAA8C;AAC9C,2CAAwC;AACxC,yDAAsD;AACtD,gDAQ0B;AAE1B,0CAAiD;AAEjD,iEAA4D;AAC5D,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AACvE,4DAAwD;AACxD,gGAA2F;AAuD3F,IAAa,aAAa,GAA1B,MAAa,aAAa;CAAG,CAAA;AAAhB,aAAa;IArDzB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,gCAAc;YACd,6BAAiB;YACjB,wBAAY;YACZ,iCAAqB;YACrB;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EACf,OAAwB,EACxB,qBAAmE,EACnE,YAA2B,EAC3B,EAAE;oBACF,MAAM,UAAU,GAAG,IAAI,6BAAkB,CACvC,OAAO,EACP,qBAAqB,EACrB,YAAY,CACb,CAAC;oBACF,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,iCAAqB,EAAE,6BAAa,CAAC;aACnE;YACD,gCAAc;YACd,yCAAkB;YAClB;gBACE,OAAO,EAAE,qCAAgB;gBACzB,UAAU,EAAE,GAAG,EAAE;oBACf,IAAI,6BAAY,EAAE;wBAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;qBACjD;oBACD,OAAO,IAAI,kCAAsB,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC;gBAC1D,CAAC;aACF;YACD,sBAAU;YACV,sBAAU;YACV;gBACE,OAAO,EAAE,iBAAiB;gBAC1B,QAAQ,EAAE,gCAAc;aACzB;YACD,8BAAa;YACb;gBACE,OAAO,EAAE,oDAAwB;gBACjC,UAAU,EAAE,GAAG,EAAE;oBACf,IAAI,6BAAY,EAAE;wBAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;qBACjD;oBACD,OAAO,IAAI,iEAA8B,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC;gBAClE,CAAC;aACF;SACF;QACD,OAAO,EAAE,CAAC,wBAAY,CAAC;KACxB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { isMainThread } from 'worker_threads';\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n ApiService,\n StoreService,\n PoiService,\n MmrService,\n ConnectionPoolService,\n StoreCacheService,\n WorkerDynamicDsService,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum';\nimport { EthereumApiConnection } from '../ethereum/api.connection';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\nimport { WorkerService } from './worker/worker.service';\nimport { WorkerUnfinalizedBlocksService } from './worker/worker.unfinalizedBlocks.service';\n\n@Module({\n providers: [\n IndexerManager,\n StoreCacheService,\n StoreService,\n ConnectionPoolService,\n {\n provide: ApiService,\n useFactory: async (\n project: SubqueryProject,\n connectionPoolService: ConnectionPoolService<EthereumApiConnection>,\n eventEmitter: EventEmitter2,\n ) => {\n const apiService = new EthereumApiService(\n project,\n connectionPoolService,\n eventEmitter,\n );\n await apiService.init();\n return apiService;\n },\n inject: ['ISubqueryProject', ConnectionPoolService, EventEmitter2],\n },\n SandboxService,\n DsProcessorService,\n {\n provide: DynamicDsService,\n useFactory: () => {\n if (isMainThread) {\n throw new Error('Expected to be worker thread');\n }\n return new WorkerDynamicDsService((global as any).host);\n },\n },\n PoiService,\n MmrService,\n {\n provide: 'IProjectService',\n useClass: ProjectService,\n },\n WorkerService,\n {\n provide: UnfinalizedBlocksService,\n useFactory: () => {\n if (isMainThread) {\n throw new Error('Expected to be worker thread');\n }\n return new WorkerUnfinalizedBlocksService((global as any).host);\n },\n },\n ],\n exports: [StoreService],\n})\nexport class IndexerModule {}\n"]}
package/dist/indexer/project.service.js CHANGED
@@ -77,7 +77,6 @@ let ProjectService = class ProjectService {
77
77
  await this.ensureMetadata();
78
78
  this.dynamicDsService.init(this.storeService.storeCache.metadata);
79
79
  await this.initHotSchemaReload();
80
- await this.initHotSchemaReload();
81
80
  if (this.nodeConfig.proofOfIndex) {
82
81
  const blockOffset = await this.getMetadataBlockOffset();
83
82
  void this.setBlockOffset(Number(blockOffset));
@@ -93,7 +92,7 @@ let ProjectService = class ProjectService {
93
92
  this._startHeight = reindexedTo;
94
93
  }
95
94
  // Flush any pending operations to setup DB
96
- await this.storeService.storeCache.flushCache();
95
+ await this.storeService.storeCache.flushCache(true);
97
96
  }
98
97
  else {
99
98
  this._schema = await this.getExistingProjectSchema();
@@ -151,7 +150,6 @@ let ProjectService = class ProjectService {
151
150
  'processedBlockCount',
152
151
  'lastFinalizedVerifiedHeight',
153
152
  'schemaMigrationCount',
154
- 'unfinalizedBlocks',
155
153
  ];
156
154
  const existing = await metadata.findMany(keys);
157
155
  const { chain, genesisHash, specName } = this.apiService.networkMeta;
package/dist/indexer/project.service.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,mDAA8C;AAC9C,2CAAoD;AACpD,yDAAsD;AACtD,gDAU0B;AAC1B,yCAAsC;AACtC,kEAIsC;AACtC,8CAAqE;AACrE,8CAA2C;AAC3C,iEAA4D;AAC5D,6DAAwD;AACxD,2EAAuE;AAEvE,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,iBAAiB,GAAG,QAAQ,CAAC;AAEnC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAKzB,YACmB,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACpB,UAAsB,EACxB,SAAoB,EACQ,OAAwB,EACpD,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAC3C,YAA2B,EAC3B,uBAAiD;QAVxC,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACpB,eAAU,GAAV,UAAU,CAAY;QACxB,cAAS,GAAT,SAAS,CAAW;QACQ,YAAO,GAAP,OAAO,CAAiB;QACpD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAC3C,iBAAY,GAAZ,YAAY,CAAe;QAC3B,4BAAuB,GAAvB,uBAAuB,CAA0B;IACxD,CAAC;IAEJ,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;IACtC,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,sEAAsE;QACtE,MAAM,IAAI,CAAC,kBAAkB,CAAC,gCAAgC,EAAE,CAAC;QACjE,8CAA8C;QAC9C,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,IAAA,2DAAyC,EACxE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,6BAAY,EAAE;YAChB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAElE,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEjC,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEjC,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;YAED,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAEhD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;gBAC3D,MAAM,CAAC,KAAK,CACV,qHAAqH,CACtH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;YAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CACzD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CACxB,CAAC;YAEF,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC7B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;aACjC;YAED,2CAA2C;YAC3C,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;SACjD;aAAM;YACL,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAErD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAA,gBAAM,EAAC,IAAI,CAAC,OAAO,EAAE,yCAAyC,CAAC,CAAC;YAChE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;SACF;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,KAAK,EAAE;YACzC,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,MAAc,CAAC;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;YAC7B,2DAA2D;YAC3D,MAAM,GAAG,iBAAiB,CAAC;SAC5B;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAE,OAA+B,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;gBACtD,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;aAC7D;SACF;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,IAAA,6BAAmB,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5D,CAAC;IACO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAEvD,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,wBAAY,CAAC,eAAe,EAC5B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QAEF,MAAM,IAAI,GAAG;YACX,qBAAqB;YACrB,aAAa;YACb,oBAAoB;YACpB,OAAO;YACP,UAAU;YACV,aAAa;YACb,aAAa;YACb,qBAAqB;YACrB,6BAA6B;YAC7B,sBAAsB;YACtB,mBAAmB;SACX,CAAC;QAEX,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE/C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAErE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE5C,QAAQ,CAAC,OAAO,CAAC;gBACf,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBACvC,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;gBACjD,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;gBACzC,EAAE,GAAG,EAAE,oBAAoB,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE;aACpD,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;SAC1C;aAAM;YACL,+EAA+E;YAC/E,IAAA,gBAAM;YACJ,uHAAuH;YACvH,CAAC,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,mCAAI,WAAW,CAAC;gBAC/C,QAAQ,CAAC,WAAW,EACtB,sJAAsJ,CACvJ,CAAC;SACH;QACD,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE;YAC5B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;SAC9B;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;SACpC;QAED,+GAA+G;QAC/G,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE;YAClE,QAAQ,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;SACxC;QAED,IAAI,QAAQ,CAAC,kBAAkB,KAAK,cAAc,EAAE;YAClD,QAAQ,CAAC,GAAG,CAAC,oBAAoB,EAAE,cAAc,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC;SACzC;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,4BAA4B,EAAE,CAAC,CAAC;SAClE;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC3E,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,WAAmB,CAAC;QACxB,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACrE,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;SAC/C;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACnD;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,IACE,IAAI,CAAC,YAAY,KAAK,SAAS;YAC/B,MAAM,KAAK,IAAI;YACf,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,CAAC,EACb;YACA,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/D,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,2BAA2B,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,4BAA4B;QAClC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAC/B,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEhE,OAAO,IAAA,iBAAO,EACZ,IAAI,CAAC,4BAA4B,EAAE,EACnC,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,uBAAuB,EAC5B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CAEf,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACzC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;QAEtE,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,CAAC,MAAM,CAC/C,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,WAAW,CACrC,CAAC;IACJ,CAAC;CACF,CAAA;AAxRY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAYR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCALU,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACR,sBAAU;QACb,qBAAS;QACiB,iCAAe;QACtC,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QAC7B,6BAAa;QACF,oDAAwB;GAhBhD,cAAc,CAwR1B;AAxRY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n ApiService,\n IProjectService,\n NodeConfig,\n IndexerEvent,\n StoreService,\n PoiService,\n MmrService,\n getLogger,\n getExistingProjectSchema,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport {\n generateTimestampReferenceForBlockFilters,\n SubqlProjectDs,\n SubqueryProject,\n} from '../configure/SubqueryProject';\nimport { initDbSchema, initHotSchemaReload } from '../utils/project';\nimport { reindex } from '../utils/reindex';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst DEFAULT_DB_SCHEMA = 'public';\n\nconst logger = getLogger('Project');\n\n@Injectable()\nexport class ProjectService implements IProjectService {\n private _schema: string;\n private _startHeight: number;\n private _blockOffset: number;\n\n constructor(\n private readonly dsProcessorService: DsProcessorService,\n private readonly apiService: ApiService,\n private readonly poiService: PoiService,\n protected readonly mmrService: MmrService,\n private readonly sequelize: Sequelize,\n @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n private readonly storeService: StoreService,\n private readonly nodeConfig: NodeConfig,\n private readonly dynamicDsService: DynamicDsService,\n private eventEmitter: EventEmitter2,\n private unfinalizedBlockService: UnfinalizedBlocksService,\n ) {}\n\n get schema(): string {\n return this._schema;\n }\n\n get dataSources(): SubqlProjectDs[] {\n return this.project.dataSources;\n }\n\n get blockOffset(): number {\n return this._blockOffset;\n }\n\n get startHeight(): number {\n return this._startHeight;\n }\n\n get isHistorical(): boolean {\n return this.storeService.historical;\n }\n\n private async getExistingProjectSchema(): Promise<string> {\n return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n }\n\n async init(): Promise<void> {\n // Used to load assets into DS-processor, has to be done in any thread\n await this.dsProcessorService.validateProjectCustomDatasources();\n // Do extra work on main thread to setup stuff\n this.project.dataSources = await generateTimestampReferenceForBlockFilters(\n this.project.dataSources,\n this.apiService.api,\n );\n if (isMainThread) {\n this._schema = await this.ensureProject();\n await this.initDbSchema();\n await this.ensureMetadata();\n this.dynamicDsService.init(this.storeService.storeCache.metadata);\n\n await this.initHotSchemaReload();\n\n await this.initHotSchemaReload();\n\n if (this.nodeConfig.proofOfIndex) {\n const blockOffset = await this.getMetadataBlockOffset();\n void this.setBlockOffset(Number(blockOffset));\n await this.poiService.init();\n }\n\n this._startHeight = await this.getStartHeight();\n\n if (this.nodeConfig.unfinalizedBlocks && !this.isHistorical) {\n logger.error(\n 'Unfinalized blocks cannot be enabled without historical. You will need to reindex your project to enable historical',\n );\n process.exit(1);\n }\n\n const reindexedTo = await this.unfinalizedBlockService.init(\n this.reindex.bind(this),\n );\n\n if (reindexedTo !== undefined) {\n this._startHeight = reindexedTo;\n }\n\n // Flush any pending operations to setup DB\n await this.storeService.storeCache.flushCache();\n } else {\n this._schema = await this.getExistingProjectSchema();\n\n await this.sequelize.sync();\n\n assert(this._schema, 'Schema should be created in main thread');\n await this.initDbSchema();\n\n if (this.nodeConfig.proofOfIndex) {\n await this.poiService.init();\n }\n }\n }\n\n private async ensureProject(): Promise<string> {\n let schema = await this.getExistingProjectSchema();\n if (!schema) {\n schema = await this.createProjectSchema();\n }\n this.eventEmitter.emit(IndexerEvent.Ready, {\n value: true,\n });\n\n return schema;\n }\n\n private async createProjectSchema(): Promise<string> {\n let schema: string;\n if (this.nodeConfig.localMode) {\n // create tables in default schema if local mode is enabled\n schema = DEFAULT_DB_SCHEMA;\n } else {\n schema = this.nodeConfig.dbSchema;\n const schemas = await this.sequelize.showAllSchemas(undefined);\n if (!(schemas as unknown as string[]).includes(schema)) {\n await this.sequelize.createSchema(`\"${schema}\"`, undefined);\n }\n }\n\n return schema;\n }\n\n private async initHotSchemaReload(): Promise<void> {\n await initHotSchemaReload(this.schema, this.storeService);\n }\n private async initDbSchema(): Promise<void> {\n await initDbSchema(this.project, this.schema, this.storeService);\n }\n\n private async ensureMetadata(): Promise<void> {\n const metadata = this.storeService.storeCache.metadata;\n\n this.eventEmitter.emit(\n IndexerEvent.NetworkMetadata,\n this.apiService.networkMeta,\n );\n\n const keys = [\n 'lastProcessedHeight',\n 'blockOffset',\n 'indexerNodeVersion',\n 'chain',\n 'specName',\n 'genesisHash',\n 'startHeight',\n 'processedBlockCount',\n 'lastFinalizedVerifiedHeight',\n 'schemaMigrationCount',\n 'unfinalizedBlocks',\n ] as const;\n\n const existing = await metadata.findMany(keys);\n\n const { chain, genesisHash, specName } = this.apiService.networkMeta;\n\n if (this.project.runner) {\n const { node, query } = this.project.runner;\n\n metadata.setBulk([\n { key: 'runnerNode', value: node.name },\n { key: 'runnerNodeVersion', value: node.version },\n { key: 'runnerQuery', value: query.name },\n { key: 'runnerQueryVersion', value: query.version },\n ]);\n }\n if (!existing.genesisHash) {\n metadata.set('genesisHash', genesisHash);\n } else {\n // Check if the configured genesisHash matches the currently stored genesisHash\n assert(\n // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1\n (this.project.network.genesisHash ?? genesisHash) ===\n existing.genesisHash,\n 'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean',\n );\n }\n if (existing.chain !== chain) {\n metadata.set('chain', chain);\n }\n\n if (existing.specName !== specName) {\n metadata.set('specName', specName);\n }\n\n // If project was created before this feature, don't add the key. If it is project created after, add this key.\n if (!existing.processedBlockCount && !existing.lastProcessedHeight) {\n metadata.set('processedBlockCount', 0);\n }\n\n if (existing.indexerNodeVersion !== packageVersion) {\n metadata.set('indexerNodeVersion', packageVersion);\n }\n if (!existing.schemaMigrationCount) {\n metadata.set('schemaMigrationCount', 0);\n }\n if (!existing.startHeight) {\n metadata.set('startHeight', this.getStartBlockFromDataSources());\n }\n }\n\n private async getMetadataBlockOffset(): Promise<number | undefined> {\n return this.storeService.storeCache.metadata.find('blockOffset');\n }\n\n private async getLastProcessedHeight(): Promise<number | undefined> {\n return this.storeService.storeCache.metadata.find('lastProcessedHeight');\n }\n\n private async getStartHeight(): Promise<number> {\n let startHeight: number;\n const lastProcessedHeight = await this.getLastProcessedHeight();\n if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {\n startHeight = Number(lastProcessedHeight) + 1;\n } else {\n startHeight = this.getStartBlockFromDataSources();\n }\n return startHeight;\n }\n\n async setBlockOffset(offset: number): Promise<void> {\n if (\n this._blockOffset !== undefined ||\n offset === null ||\n offset === undefined ||\n isNaN(offset)\n ) {\n return;\n }\n logger.info(`set blockOffset to ${offset}`);\n this._blockOffset = offset;\n return this.mmrService.syncFileBaseFromPoi(offset).catch((err) => {\n logger.error(err, 'failed to sync poi to mmr');\n process.exit(1);\n });\n }\n\n private getStartBlockFromDataSources() {\n const startBlocksList = this.project.dataSources.map(\n (item) => item.startBlock ?? 1,\n );\n if (startBlocksList.length === 0) {\n logger.error(\n `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n );\n process.exit(1);\n } else {\n return Math.min(...startBlocksList);\n }\n }\n\n async reindex(targetBlockHeight: number): Promise<void> {\n const lastProcessedHeight = await this.getLastProcessedHeight();\n\n return reindex(\n this.getStartBlockFromDataSources(),\n await this.getMetadataBlockOffset(),\n targetBlockHeight,\n lastProcessedHeight,\n this.storeService,\n this.unfinalizedBlockService,\n this.dynamicDsService,\n this.mmrService,\n this.sequelize,\n /* Not providing force clean service, it should never be needed */\n );\n }\n\n async getAllDataSources(blockHeight: number): Promise<SubqlProjectDs[]> {\n const dynamicDs = await this.dynamicDsService.getDynamicDatasources();\n\n return [...this.dataSources, ...dynamicDs].filter(\n (ds) => ds.startBlock <= blockHeight,\n );\n }\n}\n"]}
1
+ {"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,mDAA8C;AAC9C,2CAAoD;AACpD,yDAAsD;AACtD,gDAU0B;AAC1B,yCAAsC;AACtC,kEAIsC;AACtC,8CAAqE;AACrE,8CAA2C;AAC3C,iEAA4D;AAC5D,6DAAwD;AACxD,2EAAuE;AAEvE,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,iBAAiB,GAAG,QAAQ,CAAC;AAEnC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAGpC,IAAa,cAAc,GAA3B,MAAa,cAAc;IAKzB,YACmB,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACpB,UAAsB,EACxB,SAAoB,EACQ,OAAwB,EACpD,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAC3C,YAA2B,EAC3B,uBAAiD;QAVxC,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACpB,eAAU,GAAV,UAAU,CAAY;QACxB,cAAS,GAAT,SAAS,CAAW;QACQ,YAAO,GAAP,OAAO,CAAiB;QACpD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAC3C,iBAAY,GAAZ,YAAY,CAAe;QAC3B,4BAAuB,GAAvB,uBAAuB,CAA0B;IACxD,CAAC;IAEJ,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;IACtC,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,sEAAsE;QACtE,MAAM,IAAI,CAAC,kBAAkB,CAAC,gCAAgC,EAAE,CAAC;QACjE,8CAA8C;QAC9C,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,IAAA,2DAAyC,EACxE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,6BAAY,EAAE;YAChB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAElE,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEjC,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;YAED,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAEhD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;gBAC3D,MAAM,CAAC,KAAK,CACV,qHAAqH,CACtH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;YAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CACzD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CACxB,CAAC;YAEF,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC7B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;aACjC;YAED,2CAA2C;YAC3C,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;SACrD;aAAM;YACL,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAErD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAA,gBAAM,EAAC,IAAI,CAAC,OAAO,EAAE,yCAAyC,CAAC,CAAC;YAChE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;SACF;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,KAAK,EAAE;YACzC,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,MAAc,CAAC;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;YAC7B,2DAA2D;YAC3D,MAAM,GAAG,iBAAiB,CAAC;SAC5B;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAE,OAA+B,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;gBACtD,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;aAC7D;SACF;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,IAAA,6BAAmB,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5D,CAAC;IACO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAEvD,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,wBAAY,CAAC,eAAe,EAC5B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QAEF,MAAM,IAAI,GAAG;YACX,qBAAqB;YACrB,aAAa;YACb,oBAAoB;YACpB,OAAO;YACP,UAAU;YACV,aAAa;YACb,aAAa;YACb,qBAAqB;YACrB,6BAA6B;YAC7B,sBAAsB;SACd,CAAC;QAEX,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE/C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAErE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE5C,QAAQ,CAAC,OAAO,CAAC;gBACf,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBACvC,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;gBACjD,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;gBACzC,EAAE,GAAG,EAAE,oBAAoB,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE;aACpD,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;SAC1C;aAAM;YACL,+EAA+E;YAC/E,IAAA,gBAAM;YACJ,uHAAuH;YACvH,CAAC,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,mCAAI,WAAW,CAAC;gBAC/C,QAAQ,CAAC,WAAW,EACtB,sJAAsJ,CACvJ,CAAC;SACH;QACD,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE;YAC5B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;SAC9B;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;SACpC;QAED,+GAA+G;QAC/G,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE;YAClE,QAAQ,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;SACxC;QAED,IAAI,QAAQ,CAAC,kBAAkB,KAAK,cAAc,EAAE;YAClD,QAAQ,CAAC,GAAG,CAAC,oBAAoB,EAAE,cAAc,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC;SACzC;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,4BAA4B,EAAE,CAAC,CAAC;SAClE;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC3E,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,WAAmB,CAAC;QACxB,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACrE,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;SAC/C;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACnD;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,IACE,IAAI,CAAC,YAAY,KAAK,SAAS;YAC/B,MAAM,KAAK,IAAI;YACf,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,CAAC,EACb;YACA,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/D,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,2BAA2B,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,4BAA4B;QAClC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAC/B,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEhE,OAAO,IAAA,iBAAO,EACZ,IAAI,CAAC,4BAA4B,EAAE,EACnC,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,uBAAuB,EAC5B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CAEf,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACzC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;QAEtE,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,CAAC,MAAM,CAC/C,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,WAAW,CACrC,CAAC;IACJ,CAAC;CACF,CAAA;AArRY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAYR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCALU,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACR,sBAAU;QACb,qBAAS;QACiB,iCAAe;QACtC,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QAC7B,6BAAa;QACF,oDAAwB;GAhBhD,cAAc,CAqR1B;AArRY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n ApiService,\n IProjectService,\n NodeConfig,\n IndexerEvent,\n StoreService,\n PoiService,\n MmrService,\n getLogger,\n getExistingProjectSchema,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport {\n generateTimestampReferenceForBlockFilters,\n SubqlProjectDs,\n SubqueryProject,\n} from '../configure/SubqueryProject';\nimport { initDbSchema, initHotSchemaReload } from '../utils/project';\nimport { reindex } from '../utils/reindex';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst DEFAULT_DB_SCHEMA = 'public';\n\nconst logger = getLogger('Project');\n\n@Injectable()\nexport class ProjectService implements IProjectService {\n private _schema: string;\n private _startHeight: number;\n private _blockOffset: number;\n\n constructor(\n private readonly dsProcessorService: DsProcessorService,\n private readonly apiService: ApiService,\n private readonly poiService: PoiService,\n protected readonly mmrService: MmrService,\n private readonly sequelize: Sequelize,\n @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n private readonly storeService: StoreService,\n private readonly nodeConfig: NodeConfig,\n private readonly dynamicDsService: DynamicDsService,\n private eventEmitter: EventEmitter2,\n private unfinalizedBlockService: UnfinalizedBlocksService,\n ) {}\n\n get schema(): string {\n return this._schema;\n }\n\n get dataSources(): SubqlProjectDs[] {\n return this.project.dataSources;\n }\n\n get blockOffset(): number {\n return this._blockOffset;\n }\n\n get startHeight(): number {\n return this._startHeight;\n }\n\n get isHistorical(): boolean {\n return this.storeService.historical;\n }\n\n private async getExistingProjectSchema(): Promise<string> {\n return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n }\n\n async init(): Promise<void> {\n // Used to load assets into DS-processor, has to be done in any thread\n await this.dsProcessorService.validateProjectCustomDatasources();\n // Do extra work on main thread to setup stuff\n this.project.dataSources = await generateTimestampReferenceForBlockFilters(\n this.project.dataSources,\n this.apiService.api,\n );\n if (isMainThread) {\n this._schema = await this.ensureProject();\n await this.initDbSchema();\n await this.ensureMetadata();\n this.dynamicDsService.init(this.storeService.storeCache.metadata);\n\n await this.initHotSchemaReload();\n\n if (this.nodeConfig.proofOfIndex) {\n const blockOffset = await this.getMetadataBlockOffset();\n void this.setBlockOffset(Number(blockOffset));\n await this.poiService.init();\n }\n\n this._startHeight = await this.getStartHeight();\n\n if (this.nodeConfig.unfinalizedBlocks && !this.isHistorical) {\n logger.error(\n 'Unfinalized blocks cannot be enabled without historical. You will need to reindex your project to enable historical',\n );\n process.exit(1);\n }\n\n const reindexedTo = await this.unfinalizedBlockService.init(\n this.reindex.bind(this),\n );\n\n if (reindexedTo !== undefined) {\n this._startHeight = reindexedTo;\n }\n\n // Flush any pending operations to setup DB\n await this.storeService.storeCache.flushCache(true);\n } else {\n this._schema = await this.getExistingProjectSchema();\n\n await this.sequelize.sync();\n\n assert(this._schema, 'Schema should be created in main thread');\n await this.initDbSchema();\n\n if (this.nodeConfig.proofOfIndex) {\n await this.poiService.init();\n }\n }\n }\n\n private async ensureProject(): Promise<string> {\n let schema = await this.getExistingProjectSchema();\n if (!schema) {\n schema = await this.createProjectSchema();\n }\n this.eventEmitter.emit(IndexerEvent.Ready, {\n value: true,\n });\n\n return schema;\n }\n\n private async createProjectSchema(): Promise<string> {\n let schema: string;\n if (this.nodeConfig.localMode) {\n // create tables in default schema if local mode is enabled\n schema = DEFAULT_DB_SCHEMA;\n } else {\n schema = this.nodeConfig.dbSchema;\n const schemas = await this.sequelize.showAllSchemas(undefined);\n if (!(schemas as unknown as string[]).includes(schema)) {\n await this.sequelize.createSchema(`\"${schema}\"`, undefined);\n }\n }\n\n return schema;\n }\n\n private async initHotSchemaReload(): Promise<void> {\n await initHotSchemaReload(this.schema, this.storeService);\n }\n private async initDbSchema(): Promise<void> {\n await initDbSchema(this.project, this.schema, this.storeService);\n }\n\n private async ensureMetadata(): Promise<void> {\n const metadata = this.storeService.storeCache.metadata;\n\n this.eventEmitter.emit(\n IndexerEvent.NetworkMetadata,\n this.apiService.networkMeta,\n );\n\n const keys = [\n 'lastProcessedHeight',\n 'blockOffset',\n 'indexerNodeVersion',\n 'chain',\n 'specName',\n 'genesisHash',\n 'startHeight',\n 'processedBlockCount',\n 'lastFinalizedVerifiedHeight',\n 'schemaMigrationCount',\n ] as const;\n\n const existing = await metadata.findMany(keys);\n\n const { chain, genesisHash, specName } = this.apiService.networkMeta;\n\n if (this.project.runner) {\n const { node, query } = this.project.runner;\n\n metadata.setBulk([\n { key: 'runnerNode', value: node.name },\n { key: 'runnerNodeVersion', value: node.version },\n { key: 'runnerQuery', value: query.name },\n { key: 'runnerQueryVersion', value: query.version },\n ]);\n }\n if (!existing.genesisHash) {\n metadata.set('genesisHash', genesisHash);\n } else {\n // Check if the configured genesisHash matches the currently stored genesisHash\n assert(\n // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1\n (this.project.network.genesisHash ?? genesisHash) ===\n existing.genesisHash,\n 'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean',\n );\n }\n if (existing.chain !== chain) {\n metadata.set('chain', chain);\n }\n\n if (existing.specName !== specName) {\n metadata.set('specName', specName);\n }\n\n // If project was created before this feature, don't add the key. If it is project created after, add this key.\n if (!existing.processedBlockCount && !existing.lastProcessedHeight) {\n metadata.set('processedBlockCount', 0);\n }\n\n if (existing.indexerNodeVersion !== packageVersion) {\n metadata.set('indexerNodeVersion', packageVersion);\n }\n if (!existing.schemaMigrationCount) {\n metadata.set('schemaMigrationCount', 0);\n }\n if (!existing.startHeight) {\n metadata.set('startHeight', this.getStartBlockFromDataSources());\n }\n }\n\n private async getMetadataBlockOffset(): Promise<number | undefined> {\n return this.storeService.storeCache.metadata.find('blockOffset');\n }\n\n private async getLastProcessedHeight(): Promise<number | undefined> {\n return this.storeService.storeCache.metadata.find('lastProcessedHeight');\n }\n\n private async getStartHeight(): Promise<number> {\n let startHeight: number;\n const lastProcessedHeight = await this.getLastProcessedHeight();\n if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {\n startHeight = Number(lastProcessedHeight) + 1;\n } else {\n startHeight = this.getStartBlockFromDataSources();\n }\n return startHeight;\n }\n\n async setBlockOffset(offset: number): Promise<void> {\n if (\n this._blockOffset !== undefined ||\n offset === null ||\n offset === undefined ||\n isNaN(offset)\n ) {\n return;\n }\n logger.info(`set blockOffset to ${offset}`);\n this._blockOffset = offset;\n return this.mmrService.syncFileBaseFromPoi(offset).catch((err) => {\n logger.error(err, 'failed to sync poi to mmr');\n process.exit(1);\n });\n }\n\n private getStartBlockFromDataSources() {\n const startBlocksList = this.project.dataSources.map(\n (item) => item.startBlock ?? 1,\n );\n if (startBlocksList.length === 0) {\n logger.error(\n `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n );\n process.exit(1);\n } else {\n return Math.min(...startBlocksList);\n }\n }\n\n async reindex(targetBlockHeight: number): Promise<void> {\n const lastProcessedHeight = await this.getLastProcessedHeight();\n\n return reindex(\n this.getStartBlockFromDataSources(),\n await this.getMetadataBlockOffset(),\n targetBlockHeight,\n lastProcessedHeight,\n this.storeService,\n this.unfinalizedBlockService,\n this.dynamicDsService,\n this.mmrService,\n this.sequelize,\n /* Not providing force clean service, it should never be needed */\n );\n }\n\n async getAllDataSources(blockHeight: number): Promise<SubqlProjectDs[]> {\n const dynamicDs = await this.dynamicDsService.getDynamicDatasources();\n\n return [...this.dataSources, ...dynamicDs].filter(\n (ds) => ds.startBlock <= blockHeight,\n );\n }\n}\n"]}
package/dist/indexer/sandbox.service.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"sandbox.service.js","sourceRoot":"","sources":["../../src/indexer/sandbox.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,mDAA8C;AAC9C,2CAAoD;AACpD,4DAGgC;AAChC,gDAK0B;AAG1B,kEAA+E;AAC/E,8CAAmD;AAG5C,IAAM,cAAc,GAApB,MAAM,cAAc;IAGzB,YACmB,YAA0B,EAC1B,UAAsB,EACM,OAAwB;QAFpD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QAL/D,mBAAc,GAAmC,EAAE,CAAC;IAMzD,CAAC;IAEJ,qBAAqB,CACnB,EAAkB,EAClB,GAAe,EACf,YAAkC;QAElC,MAAM,KAAK,GAAU,6BAAY;YAC/B,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE;YAC9B,CAAC,CAAC,IAAA,4BAAgB,EAAE,MAAc,CAAC,IAAI,CAAC,CAAC,CAAC,wBAAwB;QAEpE,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,SAAS,EAAE;YACd,SAAS,GAAG,IAAI,0BAAc,CAC5B;gBACE,8CAA8C;gBAC9C,KAAK;gBACL,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;gBACvB,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW;gBAC9B,KAAK;aACN,EACD,IAAI,CAAC,UAAU,CAChB,CAAC;YACF,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;SACxC;QACD,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACvC,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,kBAAkB,CAAC,EAA2B;QACpD,IAAI,IAAA,oCAAkB,EAAC,EAAE,CAAC,EAAE;YAC1B,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;SACxB;aAAM;YACL,OAAO,IAAA,yBAAe,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC3C;IACH,CAAC;CACF,CAAA;AA5CY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAOR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAFI,wBAAY;QACd,sBAAU;QACe,iCAAe;GAN5D,cAAc,CA4C1B;AA5CY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n isDatasourceV0_2_0,\n SubqlEthereumDataSource,\n} from '@subql/common-ethereum';\nimport {\n NodeConfig,\n StoreService,\n IndexerSandbox,\n hostStoreToStore,\n} from '@subql/node-core';\nimport { Store } from '@subql/types';\nimport { ApiWrapper, EthereumBlockWrapper } from '@subql/types-ethereum';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { getProjectEntry } from '../utils/project';\n\n@Injectable()\nexport class SandboxService {\n private processorCache: Record<string, IndexerSandbox> = {};\n\n constructor(\n private readonly storeService: StoreService,\n private readonly nodeConfig: NodeConfig,\n @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n ) {}\n\n getDsProcessorWrapper(\n ds: SubqlProjectDs,\n api: ApiWrapper,\n blockContent: EthereumBlockWrapper,\n ): IndexerSandbox {\n const store: Store = isMainThread\n ? this.storeService.getStore()\n : hostStoreToStore((global as any).host); // Provided in worker.ts\n\n const entry = this.getDataSourceEntry(ds);\n let processor = this.processorCache[entry];\n if (!processor) {\n processor = new IndexerSandbox(\n {\n // api: await this.apiService.getPatchedApi(),\n store,\n root: this.project.root,\n script: ds.mapping.entryScript,\n entry,\n },\n this.nodeConfig,\n );\n this.processorCache[entry] = processor;\n }\n api.freezeApi(processor, blockContent);\n return processor;\n }\n\n private getDataSourceEntry(ds: SubqlEthereumDataSource): string {\n if (isDatasourceV0_2_0(ds)) {\n return ds.mapping.file;\n } else {\n return getProjectEntry(this.project.root);\n }\n }\n}\n"]}
1
+ {"version":3,"file":"sandbox.service.js","sourceRoot":"","sources":["../../src/indexer/sandbox.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,mDAA8C;AAC9C,2CAAoD;AACpD,4DAGgC;AAChC,gDAK0B;AAG1B,kEAA+E;AAC/E,8CAAmD;AAGnD,IAAa,cAAc,GAA3B,MAAa,cAAc;IAGzB,YACmB,YAA0B,EAC1B,UAAsB,EACM,OAAwB;QAFpD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QAL/D,mBAAc,GAAmC,EAAE,CAAC;IAMzD,CAAC;IAEJ,qBAAqB,CACnB,EAAkB,EAClB,GAAe,EACf,YAAkC;QAElC,MAAM,KAAK,GAAU,6BAAY;YAC/B,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE;YAC9B,CAAC,CAAC,IAAA,4BAAgB,EAAE,MAAc,CAAC,IAAI,CAAC,CAAC,CAAC,wBAAwB;QAEpE,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,SAAS,EAAE;YACd,SAAS,GAAG,IAAI,0BAAc,CAC5B;gBACE,8CAA8C;gBAC9C,KAAK;gBACL,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;gBACvB,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW;gBAC9B,KAAK;aACN,EACD,IAAI,CAAC,UAAU,CAChB,CAAC;YACF,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;SACxC;QACD,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACvC,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,kBAAkB,CAAC,EAA2B;QACpD,IAAI,IAAA,oCAAkB,EAAC,EAAE,CAAC,EAAE;YAC1B,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;SACxB;aAAM;YACL,OAAO,IAAA,yBAAe,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC3C;IACH,CAAC;CACF,CAAA;AA5CY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAOR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAFI,wBAAY;QACd,sBAAU;QACe,iCAAe;GAN5D,cAAc,CA4C1B;AA5CY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n isDatasourceV0_2_0,\n SubqlEthereumDataSource,\n} from '@subql/common-ethereum';\nimport {\n NodeConfig,\n StoreService,\n IndexerSandbox,\n hostStoreToStore,\n} from '@subql/node-core';\nimport { Store } from '@subql/types';\nimport { ApiWrapper, EthereumBlockWrapper } from '@subql/types-ethereum';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { getProjectEntry } from '../utils/project';\n\n@Injectable()\nexport class SandboxService {\n private processorCache: Record<string, IndexerSandbox> = {};\n\n constructor(\n private readonly storeService: StoreService,\n private readonly nodeConfig: NodeConfig,\n @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n ) {}\n\n getDsProcessorWrapper(\n ds: SubqlProjectDs,\n api: ApiWrapper,\n blockContent: EthereumBlockWrapper,\n ): IndexerSandbox {\n const store: Store = isMainThread\n ? this.storeService.getStore()\n : hostStoreToStore((global as any).host); // Provided in worker.ts\n\n const entry = this.getDataSourceEntry(ds);\n let processor = this.processorCache[entry];\n if (!processor) {\n processor = new IndexerSandbox(\n {\n // api: await this.apiService.getPatchedApi(),\n store,\n root: this.project.root,\n script: ds.mapping.entryScript,\n entry,\n },\n this.nodeConfig,\n );\n this.processorCache[entry] = processor;\n }\n api.freezeApi(processor, blockContent);\n return processor;\n }\n\n private getDataSourceEntry(ds: SubqlEthereumDataSource): string {\n if (isDatasourceV0_2_0(ds)) {\n return ds.mapping.file;\n } else {\n return getProjectEntry(this.project.root);\n }\n }\n}\n"]}
package/dist/indexer/unfinalizedBlocks.service.js CHANGED
@@ -15,7 +15,6 @@ exports.UnfinalizedBlocksService = exports.METADATA_LAST_FINALIZED_PROCESSED_KEY
15
15
  const common_1 = require("@nestjs/common");
16
16
  const node_core_1 = require("@subql/node-core");
17
17
  const lodash_1 = require("lodash");
18
- // import { ApiService } from './api.service';
19
18
  const logger = (0, node_core_1.getLogger)('UnfinalizedBlocks');
20
19
  exports.METADATA_UNFINALIZED_BLOCKS_KEY = 'unfinalizedBlocks';
21
20
  exports.METADATA_LAST_FINALIZED_PROCESSED_KEY = 'lastFinalizedVerifiedHeight';
package/dist/indexer/unfinalizedBlocks.service.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"unfinalizedBlocks.service.js","sourceRoot":"","sources":["../../src/indexer/unfinalizedBlocks.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAGtC,2CAA4C;AAC5C,gDAK0B;AAE1B,mCAA8B;AAE9B,8CAA8C;AAE9C,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,CAAC,CAAC;AAEjC,QAAA,+BAA+B,GAAG,mBAAmB,CAAC;AACtD,QAAA,qCAAqC,GAChD,6BAA6B,CAAC;AAEhC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAU3B,IAAM,wBAAwB,GAA9B,MAAM,wBAAwB;IAKnC,YACmB,UAAsB,EACtB,UAAsB,EACtB,UAA6B;QAF7B,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAmB;IAC7C,CAAC;IAEJ,KAAK,CAAC,IAAI,CACR,OAAgD;QAEhD,MAAM,CAAC,IAAI,CACT,yBACE,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAClD,EAAE,CACH,CAAC;QACF,qBAAqB;QACrB,IAAI,CAAC,iBAAiB,GAAG,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACnE,IAAI,CAAC,sBAAsB,GAAG,MAAM,IAAI,CAAC,8BAA8B,EAAE,CAAC;QAC1E,IAAI,CAAC,eAAe,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC1D,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CACzC,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;YACvE,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YAC7C,6CAA6C;YAE7C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE/D,IAAI,YAAY,KAAK,IAAI,EAAE;gBACzB,MAAM,CAAC,IAAI,CACT,yGAAyG,YAAY,GAAG,CACzH,CAAC;gBACF,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,8BAA8B,YAAY,GAAG,CAAC,CAAC;gBAC3D,OAAO,YAAY,CAAC;aACrB;iBAAM;gBACL,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC9B,IAAI,CAAC,gCAAgC,EAAE,CAAC;aACzC;SACF;IACH,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,KAAgC;QAEhC,IAAI,KAAK,EAAE;YACT,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;SACzD;QAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5C,IAAI,CAAC,YAAY,EAAE;YACjB,iDAAiD;YACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;SAC7B;aAAM;YACL,uDAAuD;YACvD,OAAO,IAAI,CAAC,4BAA4B,CAAC,YAAY,CAAC,CAAC;SACxD;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAY,GAAG;QACb,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7B,CAAC;IAED,IAAY,oBAAoB;QAC9B,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,sBAAsB,CAAC,MAAa;QAClC,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE;YACtE,OAAO;SACR;QACD,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC;IAChC,CAAC;IAEO,wBAAwB,CAAC,WAAmB,EAAE,IAAY;QAChE,IAAI,WAAW,IAAI,IAAI,CAAC,oBAAoB;YAAE,OAAO;QAErD,eAAe;QACf,IACE,IAAI,CAAC,iBAAiB,CAAC,MAAM;YAC7B,IAAA,aAAI,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,WAAW,EACnD;YACA,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC;QAEjD,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACrD,CAAC;IAEO,oBAAoB;QAC1B,IACE,IAAI,CAAC,sBAAsB,KAAK,SAAS;YACzC,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,oBAAoB,EACvD;YACA,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAChD,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAChE,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,oBAAoB,CAAC;IAC1D,CAAC;IAED,qEAAqE;IAC7D,eAAe,CAAC,WAAmB;QACzC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CACpD,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,GAAG,WAAW,CACnC,CAAC;IACJ,CAAC;IAED,yCAAyC;IACjC,gBAAgB,CACtB,WAAmB;QAEnB,oDAAoD;QACpD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,yBAAyB;aACjE,OAAO,EAAE,CAAC,6CAA6C;aACvD,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,WAAW,CAAC,CAAC;QACvE,IAAI,MAAM,EAAE;YACV,MAAM,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC;YACvC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,CAAC;SACvD;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,gFAAgF;IACxE,KAAK,CAAC,SAAS;QACrB,MAAM,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAC/C,IAAI,CAAC,oBAAoB,CAC1B,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,mBAAmB,EAAE;YACxB,OAAO;SACR;QAED,4CAA4C;QAC5C,IAAI,mBAAmB,CAAC,WAAW,KAAK,IAAI,CAAC,oBAAoB,EAAE;YACjE,IAAI,mBAAmB,CAAC,IAAI,KAAK,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE;gBAC1D,MAAM,CAAC,IAAI,CACT,oDAAoD,mBAAmB,CAAC,WAAW,cAAc,mBAAmB,CAAC,IAAI,oBAAoB,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CACzK,CAAC;gBACF,OAAO,IAAI,CAAC,eAAe,CAAC;aAC7B;SACF;aAAM;YACL,2CAA2C;YAC3C,IAAI,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC;YAClC;;;;eAIG;YACH,IACE,MAAM,CAAC,MAAM,GAAG,mBAAmB,CAAC,WAAW;gBAC/C,qBAAqB,EACrB;gBACA,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC5C,mBAAmB,CAAC,WAAW,CAChC,CAAC;aACH;iBAAM;gBACL,OAAO,mBAAmB,CAAC,WAAW,KAAK,MAAM,CAAC,MAAM,EAAE;oBACxD,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;iBACnE;aACF;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,mBAAmB,CAAC,IAAI,EAAE;gBAC5C,MAAM,CAAC,IAAI,CACT,oDAAoD,mBAAmB,CAAC,WAAW,cAAc,mBAAmB,CAAC,IAAI,oBAAoB,MAAM,CAAC,IAAI,EAAE,CAC3J,CAAC;gBACF,OAAO,MAAM,CAAC;aACf;SACF;QAED,OAAO;IACT,CAAC;IAEO,KAAK,CAAC,4BAA4B,CACxC,YAAmB;QAEnB,MAAM,oBAAoB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CACxD,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CACpB,MAAM,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,oBAAoB,CACvD,CAAC;QAEF,IAAI,cAAc,GAAG,YAAY,CAAC;QAElC,kEAAkE;QAClE,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,oBAAoB,CAAC,OAAO,EAAE,EAAE;YAC1D,IAAI,IAAI,KAAK,cAAc,CAAC,IAAI,IAAI,IAAI,KAAK,cAAc,CAAC,UAAU,EAAE;gBACtE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;aACtB;YAED,qBAAqB;YACrB,cAAc,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CACpD,cAAc,CAAC,UAAU,CAC1B,CAAC;SACH;QAED,OAAO,IAAI,CAAC,sBAAsB,CAAC;IACrC,CAAC;IAEO,qBAAqB,CAAC,iBAAoC;QAChE,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,uCAA+B,EAC/B,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAClC,CAAC;IACJ,CAAC;IAEO,+BAA+B,CAAC,MAAc;QACpD,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,6CAAqC,EACrC,MAAM,CACP,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,uCAA+B,EAAE,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED,gCAAgC;QAC9B,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,6CAAqC,EACrC,IAAI,CACL,CAAC;IACJ,CAAC;IAED,+BAA+B;IAC/B,KAAK,CAAC,4BAA4B;QAChC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAC7C,uCAA+B,CAChC,CAAC;QACF,IAAI,GAAG,EAAE;YACP,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAsB,CAAC;SAC7C;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,8BAA8B;QAClC,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,6CAAqC,CAAC,CAAC;IAC9E,CAAC;CACF,CAAA;AAvPY,wBAAwB;IADpC,IAAA,mBAAU,GAAE;qCAOoB,sBAAU;QACV,sBAAU;QACV,6BAAiB;GARrC,wBAAwB,CAuPpC;AAvPY,4DAAwB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Block } from '@ethersproject/abstract-provider';\nimport { Injectable } from '@nestjs/common';\nimport {\n ApiService,\n getLogger,\n NodeConfig,\n StoreCacheService,\n} from '@subql/node-core';\nimport { EthereumBlock } from '@subql/types-ethereum';\nimport { last } from 'lodash';\nimport { EthereumApi } from '../ethereum';\n// import { ApiService } from './api.service';\n\nconst logger = getLogger('UnfinalizedBlocks');\n\nexport const METADATA_UNFINALIZED_BLOCKS_KEY = 'unfinalizedBlocks';\nexport const METADATA_LAST_FINALIZED_PROCESSED_KEY =\n 'lastFinalizedVerifiedHeight';\n\nconst UNFINALIZED_THRESHOLD = 200;\n\ntype UnfinalizedBlocks = [blockHeight: number, blockHash: string][];\nexport interface IUnfinalizedBlocksService {\n processUnfinalizedBlocks(\n block: EthereumBlock | undefined,\n ): Promise<number | null>;\n}\n\n@Injectable()\nexport class UnfinalizedBlocksService implements IUnfinalizedBlocksService {\n private unfinalizedBlocks: UnfinalizedBlocks;\n private finalizedHeader: Block;\n private lastCheckedBlockHeight: number;\n\n constructor(\n private readonly apiService: ApiService,\n private readonly nodeConfig: NodeConfig,\n private readonly storeCache: StoreCacheService,\n ) {}\n\n async init(\n reindex: (targetHeight: number) => Promise<void>,\n ): Promise<number | undefined> {\n logger.info(\n `Unfinalized blocks is ${\n this.nodeConfig.unfinalizedBlocks ? 'enabled' : 'disabled'\n }`,\n );\n // unfinalized blocks\n this.unfinalizedBlocks = await this.getMetadataUnfinalizedBlocks();\n this.lastCheckedBlockHeight = await this.getLastFinalizedVerifiedHeight();\n this.finalizedHeader = await this.api.getBlockByHeightOrHash(\n await this.api.getFinalizedBlockHeight(),\n );\n\n if (!this.nodeConfig.unfinalizedBlocks && this.unfinalizedBlocks.length) {\n logger.info('Processing unfinalized blocks');\n // Validate any previously unfinalized blocks\n\n const rewindHeight = await this.processUnfinalizedBlocks(null);\n\n if (rewindHeight !== null) {\n logger.info(\n `Found un-finalized blocks from previous indexing but unverified, rolling back to last finalized block ${rewindHeight} `,\n );\n await reindex(rewindHeight);\n logger.info(`Successful rewind to block ${rewindHeight}!`);\n return rewindHeight;\n } else {\n this.resetUnfinalizedBlocks();\n this.resetLastFinalizedVerifiedHeight();\n }\n }\n }\n\n async processUnfinalizedBlocks(\n block: EthereumBlock | undefined,\n ): Promise<number | null> {\n if (block) {\n this.registerUnfinalizedBlock(block.number, block.hash);\n }\n\n const forkedHeader = await this.hasForked();\n if (!forkedHeader) {\n // Remove blocks that are now confirmed finalized\n this.deleteFinalizedBlock();\n } else {\n // Get the last unfinalized block that is now finalized\n return this.getLastCorrectFinalizedBlock(forkedHeader);\n }\n\n return null;\n }\n\n private get api(): EthereumApi {\n return this.apiService.api;\n }\n\n private get finalizedBlockNumber(): number {\n return this.finalizedHeader.number;\n }\n\n registerFinalizedBlock(header: Block): void {\n if (this.finalizedHeader && this.finalizedBlockNumber >= header.number) {\n return;\n }\n this.finalizedHeader = header;\n }\n\n private registerUnfinalizedBlock(blockNumber: number, hash: string): void {\n if (blockNumber <= this.finalizedBlockNumber) return;\n\n // Ensure order\n if (\n this.unfinalizedBlocks.length &&\n last(this.unfinalizedBlocks)[0] + 1 !== blockNumber\n ) {\n logger.error('Unfinalized block is not sequential');\n process.exit(1);\n }\n\n this.unfinalizedBlocks.push([blockNumber, hash]);\n\n this.saveUnfinalizedBlocks(this.unfinalizedBlocks);\n }\n\n private deleteFinalizedBlock(): void {\n if (\n this.lastCheckedBlockHeight !== undefined &&\n this.lastCheckedBlockHeight < this.finalizedBlockNumber\n ) {\n this.removeFinalized(this.finalizedBlockNumber);\n this.saveLastFinalizedVerifiedHeight(this.finalizedBlockNumber);\n this.saveUnfinalizedBlocks(this.unfinalizedBlocks);\n }\n this.lastCheckedBlockHeight = this.finalizedBlockNumber;\n }\n\n // remove any records less and equal than input finalized blockHeight\n private removeFinalized(blockHeight: number): void {\n this.unfinalizedBlocks = this.unfinalizedBlocks.filter(\n ([height]) => height > blockHeight,\n );\n }\n\n // find closest record from block heights\n private getClosestRecord(\n blockHeight: number,\n ): { blockHeight: number; hash: string } | undefined {\n // Have the block in the best block, can be verified\n const record = [...this.unfinalizedBlocks] // Copy so we can reverse\n .reverse() // Reverse the list to find the largest block\n .find(([bestBlockHeight]) => Number(bestBlockHeight) <= blockHeight);\n if (record) {\n const [bestBlockHeight, hash] = record;\n return { blockHeight: Number(bestBlockHeight), hash };\n }\n return undefined;\n }\n\n // check unfinalized blocks for a fork, returns the header where a fork happened\n private async hasForked(): Promise<Block | undefined> {\n const lastVerifiableBlock = this.getClosestRecord(\n this.finalizedBlockNumber,\n );\n\n // No unfinalized blocks\n if (!lastVerifiableBlock) {\n return;\n }\n\n // Unfinalized blocks beyond finalized block\n if (lastVerifiableBlock.blockHeight === this.finalizedBlockNumber) {\n if (lastVerifiableBlock.hash !== this.finalizedHeader.hash) {\n logger.warn(\n `Block fork found, enqueued un-finalized block at ${lastVerifiableBlock.blockHeight} with hash ${lastVerifiableBlock.hash}, actual hash is ${this.finalizedHeader.hash}`,\n );\n return this.finalizedHeader;\n }\n } else {\n // Unfinalized blocks below finalized block\n let header = this.finalizedHeader;\n /*\n * Iterate back through parent hashes until we get the header with the matching height\n * We use headers here rather than getBlockHash because of potential caching issues on the rpc\n * If we're off by a large number of blocks we can optimise by getting the block hash directly\n */\n if (\n header.number - lastVerifiableBlock.blockHeight >\n UNFINALIZED_THRESHOLD\n ) {\n header = await this.api.getBlockByHeightOrHash(\n lastVerifiableBlock.blockHeight,\n );\n } else {\n while (lastVerifiableBlock.blockHeight !== header.number) {\n header = await this.api.getBlockByHeightOrHash(header.parentHash);\n }\n }\n\n if (header.hash !== lastVerifiableBlock.hash) {\n logger.warn(\n `Block fork found, enqueued un-finalized block at ${lastVerifiableBlock.blockHeight} with hash ${lastVerifiableBlock.hash}, actual hash is ${header.hash}`,\n );\n return header;\n }\n }\n\n return;\n }\n\n private async getLastCorrectFinalizedBlock(\n forkedHeader: Block,\n ): Promise<number | undefined> {\n const bestVerifiableBlocks = this.unfinalizedBlocks.filter(\n ([bestBlockHeight]) =>\n Number(bestBlockHeight) <= this.finalizedBlockNumber,\n );\n\n let checkingHeader = forkedHeader;\n\n // Work backwards through the blocks until we find a matching hash\n for (const [block, hash] of bestVerifiableBlocks.reverse()) {\n if (hash === checkingHeader.hash || hash === checkingHeader.parentHash) {\n return Number(block);\n }\n\n // Get the new parent\n checkingHeader = await this.api.getBlockByHeightOrHash(\n checkingHeader.parentHash,\n );\n }\n\n return this.lastCheckedBlockHeight;\n }\n\n private saveUnfinalizedBlocks(unfinalizedBlocks: UnfinalizedBlocks): void {\n return this.storeCache.metadata.set(\n METADATA_UNFINALIZED_BLOCKS_KEY,\n JSON.stringify(unfinalizedBlocks),\n );\n }\n\n private saveLastFinalizedVerifiedHeight(height: number): void {\n return this.storeCache.metadata.set(\n METADATA_LAST_FINALIZED_PROCESSED_KEY,\n height,\n );\n }\n\n resetUnfinalizedBlocks(): void {\n this.storeCache.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, '[]');\n this.unfinalizedBlocks = [];\n }\n\n resetLastFinalizedVerifiedHeight(): void {\n return this.storeCache.metadata.set(\n METADATA_LAST_FINALIZED_PROCESSED_KEY,\n null,\n );\n }\n\n //string should be jsonb object\n async getMetadataUnfinalizedBlocks(): Promise<UnfinalizedBlocks> {\n const val = await this.storeCache.metadata.find(\n METADATA_UNFINALIZED_BLOCKS_KEY,\n );\n if (val) {\n return JSON.parse(val) as UnfinalizedBlocks;\n }\n return [];\n }\n\n async getLastFinalizedVerifiedHeight(): Promise<number | undefined> {\n return this.storeCache.metadata.find(METADATA_LAST_FINALIZED_PROCESSED_KEY);\n }\n}\n"]}
1
+ {"version":3,"file":"unfinalizedBlocks.service.js","sourceRoot":"","sources":["../../src/indexer/unfinalizedBlocks.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAGtC,2CAA4C;AAC5C,gDAK0B;AAE1B,mCAA8B;AAG9B,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,CAAC,CAAC;AAEjC,QAAA,+BAA+B,GAAG,mBAAmB,CAAC;AACtD,QAAA,qCAAqC,GAChD,6BAA6B,CAAC;AAEhC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAUlC,IAAa,wBAAwB,GAArC,MAAa,wBAAwB;IAKnC,YACmB,UAAsB,EACtB,UAAsB,EACtB,UAA6B;QAF7B,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAmB;IAC7C,CAAC;IAEJ,KAAK,CAAC,IAAI,CACR,OAAgD;QAEhD,MAAM,CAAC,IAAI,CACT,yBACE,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAClD,EAAE,CACH,CAAC;QACF,qBAAqB;QACrB,IAAI,CAAC,iBAAiB,GAAG,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACnE,IAAI,CAAC,sBAAsB,GAAG,MAAM,IAAI,CAAC,8BAA8B,EAAE,CAAC;QAC1E,IAAI,CAAC,eAAe,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC1D,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CACzC,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;YACvE,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YAC7C,6CAA6C;YAE7C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE/D,IAAI,YAAY,KAAK,IAAI,EAAE;gBACzB,MAAM,CAAC,IAAI,CACT,yGAAyG,YAAY,GAAG,CACzH,CAAC;gBACF,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,8BAA8B,YAAY,GAAG,CAAC,CAAC;gBAC3D,OAAO,YAAY,CAAC;aACrB;iBAAM;gBACL,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC9B,IAAI,CAAC,gCAAgC,EAAE,CAAC;aACzC;SACF;IACH,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,KAAgC;QAEhC,IAAI,KAAK,EAAE;YACT,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;SACzD;QAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5C,IAAI,CAAC,YAAY,EAAE;YACjB,iDAAiD;YACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;SAC7B;aAAM;YACL,uDAAuD;YACvD,OAAO,IAAI,CAAC,4BAA4B,CAAC,YAAY,CAAC,CAAC;SACxD;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAY,GAAG;QACb,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7B,CAAC;IAED,IAAY,oBAAoB;QAC9B,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,sBAAsB,CAAC,MAAa;QAClC,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE;YACtE,OAAO;SACR;QACD,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC;IAChC,CAAC;IAEO,wBAAwB,CAAC,WAAmB,EAAE,IAAY;QAChE,IAAI,WAAW,IAAI,IAAI,CAAC,oBAAoB;YAAE,OAAO;QAErD,eAAe;QACf,IACE,IAAI,CAAC,iBAAiB,CAAC,MAAM;YAC7B,IAAA,aAAI,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,WAAW,EACnD;YACA,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACrD,CAAC;IAEO,oBAAoB;QAC1B,IACE,IAAI,CAAC,sBAAsB,KAAK,SAAS;YACzC,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,oBAAoB,EACvD;YACA,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAChD,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAChE,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,oBAAoB,CAAC;IAC1D,CAAC;IAED,qEAAqE;IAC7D,eAAe,CAAC,WAAmB;QACzC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CACpD,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,GAAG,WAAW,CACnC,CAAC;IACJ,CAAC;IAED,yCAAyC;IACjC,gBAAgB,CACtB,WAAmB;QAEnB,oDAAoD;QACpD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,yBAAyB;aACjE,OAAO,EAAE,CAAC,6CAA6C;aACvD,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,WAAW,CAAC,CAAC;QACvE,IAAI,MAAM,EAAE;YACV,MAAM,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC;YACvC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,CAAC;SACvD;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,gFAAgF;IACxE,KAAK,CAAC,SAAS;QACrB,MAAM,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAC/C,IAAI,CAAC,oBAAoB,CAC1B,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,mBAAmB,EAAE;YACxB,OAAO;SACR;QAED,4CAA4C;QAC5C,IAAI,mBAAmB,CAAC,WAAW,KAAK,IAAI,CAAC,oBAAoB,EAAE;YACjE,IAAI,mBAAmB,CAAC,IAAI,KAAK,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE;gBAC1D,MAAM,CAAC,IAAI,CACT,oDAAoD,mBAAmB,CAAC,WAAW,cAAc,mBAAmB,CAAC,IAAI,oBAAoB,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CACzK,CAAC;gBACF,OAAO,IAAI,CAAC,eAAe,CAAC;aAC7B;SACF;aAAM;YACL,2CAA2C;YAC3C,IAAI,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC;YAClC;;;;eAIG;YACH,IACE,MAAM,CAAC,MAAM,GAAG,mBAAmB,CAAC,WAAW;gBAC/C,qBAAqB,EACrB;gBACA,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC5C,mBAAmB,CAAC,WAAW,CAChC,CAAC;aACH;iBAAM;gBACL,OAAO,mBAAmB,CAAC,WAAW,KAAK,MAAM,CAAC,MAAM,EAAE;oBACxD,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;iBACnE;aACF;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,mBAAmB,CAAC,IAAI,EAAE;gBAC5C,MAAM,CAAC,IAAI,CACT,oDAAoD,mBAAmB,CAAC,WAAW,cAAc,mBAAmB,CAAC,IAAI,oBAAoB,MAAM,CAAC,IAAI,EAAE,CAC3J,CAAC;gBACF,OAAO,MAAM,CAAC;aACf;SACF;QAED,OAAO;IACT,CAAC;IAEO,KAAK,CAAC,4BAA4B,CACxC,YAAmB;QAEnB,MAAM,oBAAoB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CACxD,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CACpB,MAAM,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,oBAAoB,CACvD,CAAC;QAEF,IAAI,cAAc,GAAG,YAAY,CAAC;QAElC,kEAAkE;QAClE,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,oBAAoB,CAAC,OAAO,EAAE,EAAE;YAC1D,IAAI,IAAI,KAAK,cAAc,CAAC,IAAI,IAAI,IAAI,KAAK,cAAc,CAAC,UAAU,EAAE;gBACtE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;aACtB;YAED,qBAAqB;YACrB,cAAc,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CACpD,cAAc,CAAC,UAAU,CAC1B,CAAC;SACH;QAED,OAAO,IAAI,CAAC,sBAAsB,CAAC;IACrC,CAAC;IAEO,qBAAqB,CAAC,iBAAoC;QAChE,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,uCAA+B,EAC/B,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAClC,CAAC;IACJ,CAAC;IAEO,+BAA+B,CAAC,MAAc;QACpD,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,6CAAqC,EACrC,MAAM,CACP,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,uCAA+B,EAAE,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED,gCAAgC;QAC9B,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,6CAAqC,EACrC,IAAI,CACL,CAAC;IACJ,CAAC;IAED,+BAA+B;IAC/B,KAAK,CAAC,4BAA4B;QAChC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAC7C,uCAA+B,CAChC,CAAC;QACF,IAAI,GAAG,EAAE;YACP,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAsB,CAAC;SAC7C;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,8BAA8B;QAClC,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,6CAAqC,CAAC,CAAC;IAC9E,CAAC;CACF,CAAA;AAtPY,wBAAwB;IADpC,IAAA,mBAAU,GAAE;qCAOoB,sBAAU;QACV,sBAAU;QACV,6BAAiB;GARrC,wBAAwB,CAsPpC;AAtPY,4DAAwB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Block } from '@ethersproject/abstract-provider';\nimport { Injectable } from '@nestjs/common';\nimport {\n ApiService,\n getLogger,\n NodeConfig,\n StoreCacheService,\n} from '@subql/node-core';\nimport { EthereumBlock } from '@subql/types-ethereum';\nimport { last } from 'lodash';\nimport { EthereumApi } from '../ethereum';\n\nconst logger = getLogger('UnfinalizedBlocks');\n\nexport const METADATA_UNFINALIZED_BLOCKS_KEY = 'unfinalizedBlocks';\nexport const METADATA_LAST_FINALIZED_PROCESSED_KEY =\n 'lastFinalizedVerifiedHeight';\n\nconst UNFINALIZED_THRESHOLD = 200;\n\ntype UnfinalizedBlocks = [blockHeight: number, blockHash: string][];\nexport interface IUnfinalizedBlocksService {\n processUnfinalizedBlocks(\n block: EthereumBlock | undefined,\n ): Promise<number | null>;\n}\n\n@Injectable()\nexport class UnfinalizedBlocksService implements IUnfinalizedBlocksService {\n private unfinalizedBlocks: UnfinalizedBlocks;\n private finalizedHeader: Block;\n private lastCheckedBlockHeight: number;\n\n constructor(\n private readonly apiService: ApiService,\n private readonly nodeConfig: NodeConfig,\n private readonly storeCache: StoreCacheService,\n ) {}\n\n async init(\n reindex: (targetHeight: number) => Promise<void>,\n ): Promise<number | undefined> {\n logger.info(\n `Unfinalized blocks is ${\n this.nodeConfig.unfinalizedBlocks ? 'enabled' : 'disabled'\n }`,\n );\n // unfinalized blocks\n this.unfinalizedBlocks = await this.getMetadataUnfinalizedBlocks();\n this.lastCheckedBlockHeight = await this.getLastFinalizedVerifiedHeight();\n this.finalizedHeader = await this.api.getBlockByHeightOrHash(\n await this.api.getFinalizedBlockHeight(),\n );\n\n if (!this.nodeConfig.unfinalizedBlocks && this.unfinalizedBlocks.length) {\n logger.info('Processing unfinalized blocks');\n // Validate any previously unfinalized blocks\n\n const rewindHeight = await this.processUnfinalizedBlocks(null);\n\n if (rewindHeight !== null) {\n logger.info(\n `Found un-finalized blocks from previous indexing but unverified, rolling back to last finalized block ${rewindHeight} `,\n );\n await reindex(rewindHeight);\n logger.info(`Successful rewind to block ${rewindHeight}!`);\n return rewindHeight;\n } else {\n this.resetUnfinalizedBlocks();\n this.resetLastFinalizedVerifiedHeight();\n }\n }\n }\n\n async processUnfinalizedBlocks(\n block: EthereumBlock | undefined,\n ): Promise<number | null> {\n if (block) {\n this.registerUnfinalizedBlock(block.number, block.hash);\n }\n\n const forkedHeader = await this.hasForked();\n if (!forkedHeader) {\n // Remove blocks that are now confirmed finalized\n this.deleteFinalizedBlock();\n } else {\n // Get the last unfinalized block that is now finalized\n return this.getLastCorrectFinalizedBlock(forkedHeader);\n }\n\n return null;\n }\n\n private get api(): EthereumApi {\n return this.apiService.api;\n }\n\n private get finalizedBlockNumber(): number {\n return this.finalizedHeader.number;\n }\n\n registerFinalizedBlock(header: Block): void {\n if (this.finalizedHeader && this.finalizedBlockNumber >= header.number) {\n return;\n }\n this.finalizedHeader = header;\n }\n\n private registerUnfinalizedBlock(blockNumber: number, hash: string): void {\n if (blockNumber <= this.finalizedBlockNumber) return;\n\n // Ensure order\n if (\n this.unfinalizedBlocks.length &&\n last(this.unfinalizedBlocks)[0] + 1 !== blockNumber\n ) {\n logger.error('Unfinalized block is not sequential');\n process.exit(1);\n }\n\n this.unfinalizedBlocks.push([blockNumber, hash]);\n this.saveUnfinalizedBlocks(this.unfinalizedBlocks);\n }\n\n private deleteFinalizedBlock(): void {\n if (\n this.lastCheckedBlockHeight !== undefined &&\n this.lastCheckedBlockHeight < this.finalizedBlockNumber\n ) {\n this.removeFinalized(this.finalizedBlockNumber);\n this.saveLastFinalizedVerifiedHeight(this.finalizedBlockNumber);\n this.saveUnfinalizedBlocks(this.unfinalizedBlocks);\n }\n this.lastCheckedBlockHeight = this.finalizedBlockNumber;\n }\n\n // remove any records less and equal than input finalized blockHeight\n private removeFinalized(blockHeight: number): void {\n this.unfinalizedBlocks = this.unfinalizedBlocks.filter(\n ([height]) => height > blockHeight,\n );\n }\n\n // find closest record from block heights\n private getClosestRecord(\n blockHeight: number,\n ): { blockHeight: number; hash: string } | undefined {\n // Have the block in the best block, can be verified\n const record = [...this.unfinalizedBlocks] // Copy so we can reverse\n .reverse() // Reverse the list to find the largest block\n .find(([bestBlockHeight]) => Number(bestBlockHeight) <= blockHeight);\n if (record) {\n const [bestBlockHeight, hash] = record;\n return { blockHeight: Number(bestBlockHeight), hash };\n }\n return undefined;\n }\n\n // check unfinalized blocks for a fork, returns the header where a fork happened\n private async hasForked(): Promise<Block | undefined> {\n const lastVerifiableBlock = this.getClosestRecord(\n this.finalizedBlockNumber,\n );\n\n // No unfinalized blocks\n if (!lastVerifiableBlock) {\n return;\n }\n\n // Unfinalized blocks beyond finalized block\n if (lastVerifiableBlock.blockHeight === this.finalizedBlockNumber) {\n if (lastVerifiableBlock.hash !== this.finalizedHeader.hash) {\n logger.warn(\n `Block fork found, enqueued un-finalized block at ${lastVerifiableBlock.blockHeight} with hash ${lastVerifiableBlock.hash}, actual hash is ${this.finalizedHeader.hash}`,\n );\n return this.finalizedHeader;\n }\n } else {\n // Unfinalized blocks below finalized block\n let header = this.finalizedHeader;\n /*\n * Iterate back through parent hashes until we get the header with the matching height\n * We use headers here rather than getBlockHash because of potential caching issues on the rpc\n * If we're off by a large number of blocks we can optimise by getting the block hash directly\n */\n if (\n header.number - lastVerifiableBlock.blockHeight >\n UNFINALIZED_THRESHOLD\n ) {\n header = await this.api.getBlockByHeightOrHash(\n lastVerifiableBlock.blockHeight,\n );\n } else {\n while (lastVerifiableBlock.blockHeight !== header.number) {\n header = await this.api.getBlockByHeightOrHash(header.parentHash);\n }\n }\n\n if (header.hash !== lastVerifiableBlock.hash) {\n logger.warn(\n `Block fork found, enqueued un-finalized block at ${lastVerifiableBlock.blockHeight} with hash ${lastVerifiableBlock.hash}, actual hash is ${header.hash}`,\n );\n return header;\n }\n }\n\n return;\n }\n\n private async getLastCorrectFinalizedBlock(\n forkedHeader: Block,\n ): Promise<number | undefined> {\n const bestVerifiableBlocks = this.unfinalizedBlocks.filter(\n ([bestBlockHeight]) =>\n Number(bestBlockHeight) <= this.finalizedBlockNumber,\n );\n\n let checkingHeader = forkedHeader;\n\n // Work backwards through the blocks until we find a matching hash\n for (const [block, hash] of bestVerifiableBlocks.reverse()) {\n if (hash === checkingHeader.hash || hash === checkingHeader.parentHash) {\n return Number(block);\n }\n\n // Get the new parent\n checkingHeader = await this.api.getBlockByHeightOrHash(\n checkingHeader.parentHash,\n );\n }\n\n return this.lastCheckedBlockHeight;\n }\n\n private saveUnfinalizedBlocks(unfinalizedBlocks: UnfinalizedBlocks): void {\n return this.storeCache.metadata.set(\n METADATA_UNFINALIZED_BLOCKS_KEY,\n JSON.stringify(unfinalizedBlocks),\n );\n }\n\n private saveLastFinalizedVerifiedHeight(height: number): void {\n return this.storeCache.metadata.set(\n METADATA_LAST_FINALIZED_PROCESSED_KEY,\n height,\n );\n }\n\n resetUnfinalizedBlocks(): void {\n this.storeCache.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, '[]');\n this.unfinalizedBlocks = [];\n }\n\n resetLastFinalizedVerifiedHeight(): void {\n return this.storeCache.metadata.set(\n METADATA_LAST_FINALIZED_PROCESSED_KEY,\n null,\n );\n }\n\n //string should be jsonb object\n async getMetadataUnfinalizedBlocks(): Promise<UnfinalizedBlocks> {\n const val = await this.storeCache.metadata.find(\n METADATA_UNFINALIZED_BLOCKS_KEY,\n );\n if (val) {\n return JSON.parse(val) as UnfinalizedBlocks;\n }\n return [];\n }\n\n async getLastFinalizedVerifiedHeight(): Promise<number | undefined> {\n return this.storeCache.metadata.find(METADATA_LAST_FINALIZED_PROCESSED_KEY);\n }\n}\n"]}
package/dist/indexer/worker/worker.js CHANGED
@@ -18,7 +18,6 @@ const node_worker_threads_1 = require("node:worker_threads");
18
18
  const v8_1 = require("v8");
19
19
  const core_1 = require("@nestjs/core");
20
20
  const node_core_1 = require("@subql/node-core");
21
- const dynamic_ds_service_1 = require("../dynamic-ds.service");
22
21
  const indexer_manager_1 = require("../indexer.manager");
23
22
  const worker_module_1 = require("./worker.module");
24
23
  const worker_service_1 = require("./worker.service");
@@ -53,13 +52,7 @@ async function fetchBlock(height) {
53
52
  }
54
53
  async function processBlock(height) {
55
54
  (0, assert_1.default)(workerService, 'Not initialised');
56
- const res = await workerService.processBlock(height);
57
- // Clean up the temp ds records for worker thread instance
58
- if (res.dynamicDsCreated) {
59
- const dynamicDsService = app.get(dynamic_ds_service_1.DynamicDsService);
60
- dynamicDsService.deleteTempDsRecords(height);
61
- }
62
- return res;
55
+ return workerService.processBlock(height);
63
56
  }
64
57
  // eslint-disable-next-line @typescript-eslint/require-await
65
58
  async function numFetchedBlocks() {