@subql/node-ethereum 1.10.1-0

package/dist/indexer/fetch.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.service.js","sourceRoot":"","sources":["../../src/indexer/fetch.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAA2E;AAC3E,yDAAsD;AACtD,+CAA+D;AAC/D,4DAIgC;AAChC,gDAQ0B;AAO1B,mCAA+C;AAC/C,kEAA+E;AAC/E,+DAA0D;AAC1D,4CAAkE;AAClE,6DAAyD;AACzD,6DAAwD;AAGxD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,OAAO,CAAC,CAAC;AAClC,IAAI,mBAAmB,GAAG,IAAI,CAAC;AAC/B,MAAM,yBAAyB,GAAG,KAAK,CAAC;AACxC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,SAAS,uBAAuB,CAC9B,MAAyB;IAEzB,MAAM,UAAU,GAAG,EAAE,CAAC;IACtB,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,MAAM,EAAE;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,EAAE;gBACV,SAAS;aACV;YACD,MAAM,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAA,qBAAY,EAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACxD;KACF;IACD,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,MAAiC;IAEjC,MAAM,UAAU,GAAG,EAAE,CAAC;IACtB,IAAI,MAAM,CAAC,IAAI,EAAE;QACf,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE;SACjC,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,EAAE,EAAE;QACb,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE;SAC/B,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,QAAQ,EAAE;QACnB,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,IAAA,0BAAiB,EAAC,MAAM,CAAC,QAAQ,CAAC;SAC1C,CAAC,CAAC;KACJ;IACD,OAAO;QACL,MAAM,EAAE,iBAAiB;QACzB,UAAU;KACX,CAAC;AACJ,CAAC;AAGM,IAAM,YAAY,GAAlB,MAAM,YAAY;IASvB,YACU,UAAsB,EACtB,UAAsB,EACtB,OAAwB,EACI,eAAiC,EAC7D,iBAAoC,EACpC,gBAAkC,EAClC,YAA2B,EAC3B,iBAAoC;QAPpC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,YAAO,GAAP,OAAO,CAAiB;QACI,oBAAe,GAAf,eAAe,CAAkB;QAC7D,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,iBAAY,GAAZ,YAAY,CAAe;QAC3B,sBAAiB,GAAjB,iBAAiB,CAAmB;QAdtC,eAAU,GAAG,KAAK,CAAC;QAgBzB,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,qBAAqB;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,+BAA+B;QACnC,IAAI,CAAC,yBAAyB;YAC5B,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;IACxD,CAAC;IAED,6EAA6E;IAC7E,yBAAyB;QACvB,MAAM,YAAY,GAA2B,EAAE,CAAC;QAEhD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAC7C,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,EAAE;YACnE,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IAAI,UAAgC,CAAC;gBACrC,UAAU,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC9B,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;gBACzC,IAAI,CAAC,UAAU,CAAC,MAAM;oBAAE,OAAO,EAAE,CAAC;gBAClC,QAAQ,OAAO,CAAC,IAAI,EAAE;oBACpB,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,EAAE,CAAC;oBACZ,KAAK,qCAAmB,CAAC,IAAI,CAAC,CAAC;wBAC7B,KAAK,MAAM,MAAM,IAAI,UAAyC,EAAE;4BAC9D,IACE,MAAM,CAAC,IAAI,KAAK,SAAS;gCACzB,MAAM,CAAC,EAAE,KAAK,SAAS;gCACvB,MAAM,CAAC,QAAQ,EACf;gCACA,YAAY,CAAC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC;6BACnD;iCAAM;gCACL,OAAO,EAAE,CAAC;6BACX;yBACF;wBACD,MAAM;qBACP;oBACD,KAAK,qCAAmB,CAAC,KAAK,CAAC,CAAC;wBAC9B,KAAK,MAAM,MAAM,IAAI,UAAiC,EAAE;4BACtD,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,EAAE;gCACnC,YAAY,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC;6BACpD;iCAAM;gCACL,OAAO,EAAE,CAAC;6BACX;yBACF;wBACD,MAAM;qBACP;oBACD,QAAQ;iBACT;aACF;SACF;QAED,OAAO,IAAA,eAAM,EACX,YAAY,EACZ,CAAC,IAAI,EAAE,EAAE,CACP,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAC9B,IAAA,eAAM,EAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACxC,EAAE,CACN,CAAC;IACJ,CAAC;IAED,gBAAgB;;QACd,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAC/D,IAAI,CAAC,aAAa;YAChB,CAAC,CAAC,CAAA,MAAA,IAAI,CAAC,sBAAsB,0CAAE,MAAM,CAAA;gBACrC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,WAAmB;QAC5B,IAAI,IAAI,CAAC,GAAG,EAAE;YACZ,MAAM,cAAc,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,GAAG,CAAC,GAAG,gBAAgB,CAAC;YAEjE,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;YAEpE,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAChC,oBAAoB,EACpB,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,gBAAgB,EAAE,EAAE,mBAAmB,CAAC,CACrE,CAAC;SACH;QACD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;YACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;SAClC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE9B,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAE/D,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAGD,eAAe;QACb,IAAI,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE;YACvC,MAAM,KAAK,GAAG,IAAA,4BAAgB,EAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAErE,IAAI,IAAI,CAAC,cAAc,KAAK,KAAK,EAAE;gBACjC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;aAC7B;SACF;IACH,CAAC;IAGK,AAAN,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC9D,OAAO;SACR;QACD,IAAI;YACF,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC;YACxE,IAAI,IAAI,CAAC,qBAAqB,KAAK,sBAAsB,EAAE;gBACzD,IAAI,CAAC,qBAAqB,GAAG,sBAAsB,CAAC;gBACpD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;oBAC/C,MAAM,EAAE,IAAI,CAAC,qBAAqB;iBACnC,CAAC,CAAC;aACJ;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,2CAA2C,CAAC,CAAC;SAC9D;IACH,CAAC;IAGK,AAAN,KAAK,CAAC,gBAAgB;QACpB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,OAAO;SACR;QACD,IAAI;YACF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;YACzD,IAAI,IAAI,CAAC,gBAAgB,KAAK,iBAAiB,EAAE;gBAC/C,IAAI,CAAC,gBAAgB,GAAG,iBAAiB,CAAC;gBAC1C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,SAAS,EAAE;oBAC7C,MAAM,EAAE,IAAI,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;aACJ;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,sCAAsC,CAAC,CAAC;SACzD;IACH,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,eAAuB;QACrC,MAAM,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IAClD,CAAC;IAED,UAAU;QACR,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;YACzC,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;gBAClB,SAAS;aACV;YACD,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IACE,OAAO,CAAC,IAAI,KAAK,qCAAmB,CAAC,KAAK;oBAC1C,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,MAAM,CAAC,MAAM,EACrB;oBACA,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;iBACrC;aACF;SACF;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,eAAe,CAAC,WAAmB,EAAE,SAAiB;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,WAAW,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE;YAC5C,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE;gBACpC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACtB;SACF;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,eAAuB;QAC/C,IAAI,gBAAwB,CAAC;QAC7B,IAAI,eAAuB,CAAC;QAE5B,MAAM,mBAAmB,GAAG,GAAW,EAAE;YACvC,OAAO,IAAI,CAAC,eAAe,CAAC,oBAAoB;gBAC9C,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,GAAG,CAAC;gBAC/C,CAAC,CAAC,eAAe,CAAC;QACtB,CAAC,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE;YACvB,gBAAgB,GAAG,mBAAmB,EAAE,CAAC;YAEzC,eAAe,GAAG,IAAI,CAAC,GAAG,CACxB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAC3D,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CAC5D,CAAC;YAEF,IACE,IAAI,CAAC,eAAe,CAAC,QAAQ,GAAG,eAAe;gBAC/C,gBAAgB,GAAG,IAAI,CAAC,qBAAqB,EAC7C;gBACA,MAAM,IAAA,iBAAK,EAAC,CAAC,CAAC,CAAC;gBACf,SAAS;aACV;YACD,IAAI,IAAI,CAAC,aAAa,EAAE;gBACtB,MAAM,aAAa,GAAG,gBAAgB,GAAG,yBAAyB,CAAC;gBACnE,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CACvC,gBAAgB,EAChB,aAAa,CACd,CAAC;gBACF,IAAI;oBACF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAC3D,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,IAAI,CAAC,sBAAsB,CAC5B,CAAC;oBAEF,IAAI,gBAAgB,KAAK,mBAAmB,EAAE,EAAE;wBAC9C,MAAM,CAAC,KAAK,CACV,gEAAgE,CACjE,CAAC;wBACF,SAAS;qBACV;oBAED,IACE,UAAU;wBACV,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,gBAAgB,CAAC,EACvD;wBACA,IAAI,EAAE,WAAW,EAAE,GAAG,UAAU,CAAC;wBACjC,WAAW,GAAG,WAAW;6BACtB,MAAM,CAAC,YAAY,CAAC;6BACpB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;wBACzB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;4BAC5B,iFAAiF;4BACjF,IAAI,CAAC,eAAe,CAAC,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAClD,aAAa,GAAG,CAAC,EACjB,UAAU,CAAC,SAAS,CAAC,mBAAmB,CACzC,CAAC;yBACH;6BAAM;4BACL,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,WAAW,CAAC,MAAM,EAClB,IAAI,CAAC,eAAe,CAAC,QAAQ,CAC9B,CAAC;4BACF,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;4BACjD,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;yBACjD;wBACD,SAAS,CAAC,4BAA4B;qBACvC;oBACD,iCAAiC;iBAClC;gBAAC,OAAO,CAAC,EAAE;oBACV,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBACvD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;iBACrD;aACF;YACD,sDAAsD;YACtD,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CACvC,gBAAgB,EAChB,eAAe,CAChB,CAAC;YACF,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,IAAA,cAAK,EAAC,gBAAgB,EAAE,SAAS,GAAG,CAAC,CAAC,CACvC,CAAC;SACH;IACH,CAAC;IAEO,kBAAkB,CACxB,gBAAwB,EACxB,eAAuB;QAEvB,IAAI,cAAc,GAAG,gBAAgB,GAAG,eAAe,GAAG,CAAC,CAAC;QAE5D,IAAI,cAAc,GAAG,IAAI,CAAC,qBAAqB,EAAE;YAC/C,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC;SAC7C;QACD,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,oBAAoB,CAC1B,EAAE,SAAS,EAAE,QAAQ,EAAc,EACnC,gBAAwB;QAExB,IAAI,QAAQ,CAAC,WAAW,KAAK,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE;YACtD,MAAM,CAAC,KAAK,CACV,+KAA+K,CAChL,CAAC;YACF,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;gBACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;aAClC,CAAC,CAAC;YACH,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;SACd;QACD,IAAI,QAAQ,CAAC,mBAAmB,GAAG,gBAAgB,EAAE;YACnD,MAAM,CAAC,IAAI,CACT,kEAAkE,CACnE,CAAC;YACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;SACd;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,WAAmB;QACrC,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;CACF,CAAA;AApNC;IAAC,IAAA,mBAAQ,EAAC,qBAAqB,CAAC;;;;mDAS/B;AAGK;IADL,IAAA,mBAAQ,EAAC,mBAAmB,GAAG,IAAI,CAAC;;;;yDAiBpC;AAGK;IADL,IAAA,mBAAQ,EAAC,mBAAmB,GAAG,IAAI,CAAC;;;;oDAiBpC;AArKU,YAAY;IADxB,IAAA,mBAAU,GAAE;IAcR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAHP,sBAAU;QACV,sBAAU;QACb,iCAAe,UAEL,sCAAiB;QAClB,qCAAgB;QACpB,6BAAa;QACR,4BAAiB;GAjBnC,YAAY,CA0UxB;AA1UY,oCAAY","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval, SchedulerRegistry } from '@nestjs/schedule';\nimport {\n  isCustomDs,\n  EthereumHandlerKind,\n  SubqlHandlerFilter,\n} from '@subql/common-ethereum';\nimport {\n  ApiService,\n  Dictionary,\n  delay,\n  checkMemoryUsage,\n  NodeConfig,\n  IndexerEvent,\n  getLogger,\n} from '@subql/node-core';\nimport {\n  DictionaryQueryEntry,\n  ApiWrapper,\n  EthereumLogFilter,\n  EthereumTransactionFilter,\n} from '@subql/types-ethereum';\nimport { range, sortBy, uniqBy } from 'lodash';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { calcInterval } from '../ethereum/utils.ethereum';\nimport { eventToTopic, functionToSighash } from '../utils/string';\nimport { DictionaryService } from './dictionary.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { IBlockDispatcher } from './worker/block-dispatcher.service';\n\nconst logger = getLogger('fetch');\nlet BLOCK_TIME_VARIANCE = 5000;\nconst DICTIONARY_MAX_QUERY_SIZE = 10000;\nconst CHECK_MEMORY_INTERVAL = 60000;\nconst MINIMUM_BATCH_SIZE = 5;\nconst INTERVAL_PERCENT = 0.9;\n\nfunction eventFilterToQueryEntry(\n  filter: EthereumLogFilter,\n): DictionaryQueryEntry {\n  const conditions = [];\n  if (filter.address) {\n    conditions.push({\n      field: 'address',\n      value: filter.address.toLowerCase(),\n    });\n  }\n  if (filter.topics) {\n    for (let i = 0; i < Math.min(filter.topics.length, 4); i++) {\n      const topic = filter.topics[i];\n      if (!topic) {\n        continue;\n      }\n      const field = `topics${i}`;\n      conditions.push({ field, value: eventToTopic(topic) });\n    }\n  }\n  return {\n    entity: 'evmLogs',\n    conditions,\n  };\n}\n\nfunction callFilterToQueryEntry(\n  filter: EthereumTransactionFilter,\n): DictionaryQueryEntry {\n  const conditions = [];\n  if (filter.from) {\n    conditions.push({\n      field: 'from',\n      value: filter.from.toLowerCase(),\n    });\n  }\n  if (filter.to) {\n    conditions.push({\n      field: 'to',\n      value: filter.to.toLowerCase(),\n    });\n  }\n  if (filter.function) {\n    conditions.push({\n      field: 'func',\n      value: functionToSighash(filter.function),\n    });\n  }\n  return {\n    entity: 'evmTransactions',\n    conditions,\n  };\n}\n\n@Injectable()\nexport class FetchService implements OnApplicationShutdown {\n  private latestBestHeight: number;\n  private latestFinalizedHeight: number;\n  private isShutdown = false;\n  private useDictionary: boolean;\n  private dictionaryQueryEntries?: DictionaryQueryEntry[];\n  private batchSizeScale: number;\n  private templateDynamicDatasouces: SubqlProjectDs[];\n\n  constructor(\n    private apiService: ApiService,\n    private nodeConfig: NodeConfig,\n    private project: SubqueryProject,\n    @Inject('IBlockDispatcher') private blockDispatcher: IBlockDispatcher,\n    private dictionaryService: DictionaryService,\n    private dynamicDsService: DynamicDsService,\n    private eventEmitter: EventEmitter2,\n    private schedulerRegistry: SchedulerRegistry,\n  ) {\n    this.batchSizeScale = 1;\n  }\n\n  onApplicationShutdown(): void {\n    this.isShutdown = true;\n  }\n\n  get api(): ApiWrapper {\n    return this.apiService.api;\n  }\n\n  async syncDynamicDatascourcesFromMeta(): Promise<void> {\n    this.templateDynamicDatasouces =\n      await this.dynamicDsService.getDynamicDatasources();\n  }\n\n  // TODO: if custom ds doesn't support dictionary, use baseFilter, if yes, let\n  getDictionaryQueryEntries(): DictionaryQueryEntry[] {\n    const queryEntries: DictionaryQueryEntry[] = [];\n\n    const dataSources = this.project.dataSources;\n    for (const ds of dataSources.concat(this.templateDynamicDatasouces)) {\n      for (const handler of ds.mapping.handlers) {\n        let filterList: SubqlHandlerFilter[];\n        filterList = [handler.filter];\n        filterList = filterList.filter((f) => f);\n        if (!filterList.length) return [];\n        switch (handler.kind) {\n          case EthereumHandlerKind.Block:\n            return [];\n          case EthereumHandlerKind.Call: {\n            for (const filter of filterList as EthereumTransactionFilter[]) {\n              if (\n                filter.from !== undefined ||\n                filter.to !== undefined ||\n                filter.function\n              ) {\n                queryEntries.push(callFilterToQueryEntry(filter));\n              } else {\n                return [];\n              }\n            }\n            break;\n          }\n          case EthereumHandlerKind.Event: {\n            for (const filter of filterList as EthereumLogFilter[]) {\n              if (filter.address || filter.topics) {\n                queryEntries.push(eventFilterToQueryEntry(filter));\n              } else {\n                return [];\n              }\n            }\n            break;\n          }\n          default:\n        }\n      }\n    }\n\n    return uniqBy(\n      queryEntries,\n      (item) =>\n        `${item.entity}|${JSON.stringify(\n          sortBy(item.conditions, (c) => c.field),\n        )}`,\n    );\n  }\n\n  updateDictionary(): void {\n    this.dictionaryQueryEntries = this.getDictionaryQueryEntries();\n    this.useDictionary =\n      !!this.dictionaryQueryEntries?.length &&\n      !!this.project.network.dictionary;\n  }\n\n  async init(startHeight: number): Promise<void> {\n    if (this.api) {\n      const CHAIN_INTERVAL = calcInterval(this.api) * INTERVAL_PERCENT;\n\n      BLOCK_TIME_VARIANCE = Math.min(BLOCK_TIME_VARIANCE, CHAIN_INTERVAL);\n\n      this.schedulerRegistry.addInterval(\n        'getLatestBlockHead',\n        setInterval(() => void this.getBestBlockHead(), BLOCK_TIME_VARIANCE),\n      );\n    }\n    await this.syncDynamicDatascourcesFromMeta();\n    this.updateDictionary();\n    this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n      value: Number(this.useDictionary),\n    });\n    await this.getFinalizedBlockHead();\n    await this.getBestBlockHead();\n\n    await this.blockDispatcher.init(this.resetForNewDs.bind(this));\n\n    void this.startLoop(startHeight);\n  }\n\n  @Interval(CHECK_MEMORY_INTERVAL)\n  checkBatchScale(): void {\n    if (this.nodeConfig['scale-batch-size']) {\n      const scale = checkMemoryUsage(this.batchSizeScale, this.nodeConfig);\n\n      if (this.batchSizeScale !== scale) {\n        this.batchSizeScale = scale;\n      }\n    }\n  }\n\n  @Interval(BLOCK_TIME_VARIANCE * 1000)\n  async getFinalizedBlockHead(): Promise<void> {\n    if (!this.api) {\n      logger.debug(`Skip fetch finalized block until API is ready`);\n      return;\n    }\n    try {\n      const currentFinalizedHeight = await this.api.getFinalizedBlockHeight();\n      if (this.latestFinalizedHeight !== currentFinalizedHeight) {\n        this.latestFinalizedHeight = currentFinalizedHeight;\n        this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n          height: this.latestFinalizedHeight,\n        });\n      }\n    } catch (e) {\n      logger.error(e, `Having a problem when get finalized block`);\n    }\n  }\n\n  @Interval(BLOCK_TIME_VARIANCE * 1000)\n  async getBestBlockHead(): Promise<void> {\n    if (!this.api) {\n      logger.debug(`Skip fetch best block until API is ready`);\n      return;\n    }\n    try {\n      const currentBestHeight = await this.api.getLastHeight();\n      if (this.latestBestHeight !== currentBestHeight) {\n        this.latestBestHeight = currentBestHeight;\n        this.eventEmitter.emit(IndexerEvent.BlockBest, {\n          height: this.latestBestHeight,\n        });\n      }\n    } catch (e) {\n      logger.error(e, `Having a problem when get best block`);\n    }\n  }\n\n  async startLoop(initBlockHeight: number): Promise<void> {\n    await this.fillNextBlockBuffer(initBlockHeight);\n  }\n\n  getModulos(): number[] {\n    const modulos: number[] = [];\n    for (const ds of this.project.dataSources) {\n      if (isCustomDs(ds)) {\n        continue;\n      }\n      for (const handler of ds.mapping.handlers) {\n        if (\n          handler.kind === EthereumHandlerKind.Block &&\n          handler.filter &&\n          handler.filter.modulo\n        ) {\n          modulos.push(handler.filter.modulo);\n        }\n      }\n    }\n    return modulos;\n  }\n\n  getModuloBlocks(startHeight: number, endHeight: number): number[] {\n    const modulos = this.getModulos();\n    const moduloBlocks: number[] = [];\n    for (let i = startHeight; i < endHeight; i++) {\n      if (modulos.find((m) => i % m === 0)) {\n        moduloBlocks.push(i);\n      }\n    }\n    return moduloBlocks;\n  }\n\n  async fillNextBlockBuffer(initBlockHeight: number): Promise<void> {\n    let startBlockHeight: number;\n    let scaledBatchSize: number;\n\n    const getStartBlockHeight = (): number => {\n      return this.blockDispatcher.latestBufferedHeight\n        ? this.blockDispatcher.latestBufferedHeight + 1\n        : initBlockHeight;\n    };\n\n    while (!this.isShutdown) {\n      startBlockHeight = getStartBlockHeight();\n\n      scaledBatchSize = Math.max(\n        Math.round(this.batchSizeScale * this.nodeConfig.batchSize),\n        Math.min(MINIMUM_BATCH_SIZE, this.nodeConfig.batchSize * 3),\n      );\n\n      if (\n        this.blockDispatcher.freeSize < scaledBatchSize ||\n        startBlockHeight > this.latestFinalizedHeight\n      ) {\n        await delay(1);\n        continue;\n      }\n      if (this.useDictionary) {\n        const queryEndBlock = startBlockHeight + DICTIONARY_MAX_QUERY_SIZE;\n        const moduloBlocks = this.getModuloBlocks(\n          startBlockHeight,\n          queryEndBlock,\n        );\n        try {\n          const dictionary = await this.dictionaryService.getDictionary(\n            startBlockHeight,\n            queryEndBlock,\n            scaledBatchSize,\n            this.dictionaryQueryEntries,\n          );\n\n          if (startBlockHeight !== getStartBlockHeight()) {\n            logger.debug(\n              `Queue was reset for new DS, discarding dictionary query result`,\n            );\n            continue;\n          }\n\n          if (\n            dictionary &&\n            this.dictionaryValidation(dictionary, startBlockHeight)\n          ) {\n            let { batchBlocks } = dictionary;\n            batchBlocks = batchBlocks\n              .concat(moduloBlocks)\n              .sort((a, b) => a - b);\n            if (batchBlocks.length === 0) {\n              // There we're no blocks in this query range, we can set a new height we're up to\n              this.blockDispatcher.latestBufferedHeight = Math.min(\n                queryEndBlock - 1,\n                dictionary._metadata.lastProcessedHeight,\n              );\n            } else {\n              const maxBlockSize = Math.min(\n                batchBlocks.length,\n                this.blockDispatcher.freeSize,\n              );\n              batchBlocks = batchBlocks.slice(0, maxBlockSize);\n              this.blockDispatcher.enqueueBlocks(batchBlocks);\n            }\n            continue; // skip nextBlockRange() way\n          }\n          // else use this.nextBlockRange()\n        } catch (e) {\n          logger.debug(`Fetch dictionary stopped: ${e.message}`);\n          this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n        }\n      }\n      // the original method: fill next batch size of blocks\n      const endHeight = this.nextEndBlockHeight(\n        startBlockHeight,\n        scaledBatchSize,\n      );\n      this.blockDispatcher.enqueueBlocks(\n        range(startBlockHeight, endHeight + 1),\n      );\n    }\n  }\n\n  private nextEndBlockHeight(\n    startBlockHeight: number,\n    scaledBatchSize: number,\n  ): number {\n    let endBlockHeight = startBlockHeight + scaledBatchSize - 1;\n\n    if (endBlockHeight > this.latestFinalizedHeight) {\n      endBlockHeight = this.latestFinalizedHeight;\n    }\n    return endBlockHeight;\n  }\n\n  private dictionaryValidation(\n    { _metadata: metaData }: Dictionary,\n    startBlockHeight: number,\n  ): boolean {\n    if (metaData.genesisHash !== this.api.getGenesisHash()) {\n      logger.error(\n        'The dictionary that you have specified does not match the chain you are indexing, it will be ignored. Please update your project manifest to reference the correct dictionary',\n      );\n      this.useDictionary = false;\n      this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n        value: Number(this.useDictionary),\n      });\n      this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n      return false;\n    }\n    if (metaData.lastProcessedHeight < startBlockHeight) {\n      logger.warn(\n        `Dictionary indexed block is behind current indexing block height`,\n      );\n      this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n      return false;\n    }\n    return true;\n  }\n\n  async resetForNewDs(blockHeight: number): Promise<void> {\n    await this.syncDynamicDatascourcesFromMeta();\n    this.updateDictionary();\n    this.blockDispatcher.flushQueue(blockHeight);\n  }\n}\n"]}

package/dist/indexer/indexer.manager.d.ts

@@ -0,0 +1,36 @@
+import { ApiService, StoreService, PoiService, NodeConfig } from '@subql/node-core';
+import { EthereumBlockWrapper } from '@subql/types-ethereum';
+import { Sequelize } from 'sequelize';
+import { SubqueryProject } from '../configure/SubqueryProject';
+import { DsProcessorService } from './ds-processor.service';
+import { DynamicDsService } from './dynamic-ds.service';
+import { ProjectService } from './project.service';
+import { SandboxService } from './sandbox.service';
+export declare class IndexerManager {
+    private storeService;
+    private apiService;
+    private poiService;
+    private sequelize;
+    private project;
+    private nodeConfig;
+    private sandboxService;
+    private dynamicDsService;
+    private dsProcessorService;
+    private projectService;
+    private api;
+    private filteredDataSources;
+    constructor(storeService: StoreService, apiService: ApiService, poiService: PoiService, sequelize: Sequelize, project: SubqueryProject, nodeConfig: NodeConfig, sandboxService: SandboxService, dynamicDsService: DynamicDsService, dsProcessorService: DsProcessorService, projectService: ProjectService);
+    indexBlock(blockContent: EthereumBlockWrapper): Promise<{
+        dynamicDsCreated: boolean;
+        operationHash: Uint8Array;
+    }>;
+    start(): Promise<void>;
+    private filterDataSources;
+    private indexBlockData;
+    private indexBlockContent;
+    private indexExtrinsic;
+    private indexEvent;
+    private indexData;
+    private filterCustomDsHandlers;
+    private transformAndExecuteCustomDs;
+}

package/dist/indexer/indexer.manager.js

@@ -0,0 +1,256 @@
+"use strict";
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IndexerManager = void 0;
+const common_1 = require("@nestjs/common");
+const util_1 = require("@polkadot/util");
+const common_ethereum_1 = require("@subql/common-ethereum");
+const node_core_1 = require("@subql/node-core");
+const sequelize_1 = require("sequelize");
+const SubqueryProject_1 = require("../configure/SubqueryProject");
+const block_ethereum_1 = require("../ethereum/block.ethereum");
+const yargs_1 = require("../yargs");
+const ds_processor_service_1 = require("./ds-processor.service");
+const dynamic_ds_service_1 = require("./dynamic-ds.service");
+const project_service_1 = require("./project.service");
+const sandbox_service_1 = require("./sandbox.service");
+const NULL_MERKEL_ROOT = (0, util_1.hexToU8a)('0x00');
+const logger = (0, node_core_1.getLogger)('indexer');
+let IndexerManager = class IndexerManager {
+    constructor(storeService, apiService, poiService, sequelize, project, nodeConfig, sandboxService, dynamicDsService, dsProcessorService, projectService) {
+        this.storeService = storeService;
+        this.apiService = apiService;
+        this.poiService = poiService;
+        this.sequelize = sequelize;
+        this.project = project;
+        this.nodeConfig = nodeConfig;
+        this.sandboxService = sandboxService;
+        this.dynamicDsService = dynamicDsService;
+        this.dsProcessorService = dsProcessorService;
+        this.projectService = projectService;
+        logger.info('indexer manager start');
+        this.api = this.apiService.api;
+    }
+    async indexBlock(blockContent) {
+        const { blockHeight } = blockContent;
+        let dynamicDsCreated = false;
+        const tx = await this.sequelize.transaction();
+        this.storeService.setTransaction(tx);
+        this.storeService.setBlockHeight(blockHeight);
+        let operationHash = NULL_MERKEL_ROOT;
+        let poiBlockHash;
+        try {
+            this.filteredDataSources = this.filterDataSources(blockHeight);
+            const datasources = this.filteredDataSources.concat(...(await this.dynamicDsService.getDynamicDatasources()));
+            await this.indexBlockData(blockContent, datasources, 
+            // eslint-disable-next-line @typescript-eslint/require-await
+            async (ds) => {
+                const vm = this.sandboxService.getDsProcessorWrapper(ds, this.api, blockContent);
+                // Inject function to create ds into vm
+                vm.freeze(async (templateName, args) => {
+                    const newDs = await this.dynamicDsService.createDynamicDatasource({
+                        templateName,
+                        args,
+                        startBlock: blockHeight,
+                    }, tx);
+                    // Push the newly created dynamic ds to be processed this block on any future extrinsics/events
+                    datasources.push(newDs);
+                    dynamicDsCreated = true;
+                }, 'createDynamicDatasource');
+                return vm;
+            });
+            await this.storeService.setMetadataBatch([
+                { key: 'lastProcessedHeight', value: blockHeight },
+                { key: 'lastProcessedTimestamp', value: Date.now() },
+            ], { transaction: tx });
+            // Db Metadata increase BlockCount, in memory ref to block-dispatcher _processedBlockCount
+            await this.storeService.incrementBlockCount(tx);
+            // Need calculate operationHash to ensure correct offset insert all time
+            operationHash = this.storeService.getOperationMerkleRoot();
+            if (this.nodeConfig.proofOfIndex) {
+                //check if operation is null, then poi will not be inserted
+                if (!(0, util_1.u8aEq)(operationHash, NULL_MERKEL_ROOT)) {
+                    const poiBlock = node_core_1.PoiBlock.create(blockHeight, blockContent.block.hash, operationHash, await this.poiService.getLatestPoiBlockHash(), this.project.id);
+                    poiBlockHash = poiBlock.hash;
+                    await this.storeService.setPoi(poiBlock, { transaction: tx });
+                    this.poiService.setLatestPoiBlockHash(poiBlockHash);
+                    await this.storeService.setMetadataBatch([{ key: 'lastPoiHeight', value: blockHeight }], { transaction: tx });
+                }
+            }
+        }
+        catch (e) {
+            await tx.rollback();
+            throw e;
+        }
+        await tx.commit();
+        return {
+            dynamicDsCreated,
+            operationHash,
+        };
+    }
+    async start() {
+        await this.projectService.init();
+    }
+    filterDataSources(processedHeight) {
+        let filteredDs = this.project.dataSources;
+        if (filteredDs.length === 0) {
+            logger.error(`Did not find any dataSource match with network specName ${this.api.getSpecName()}`);
+            process.exit(1);
+        }
+        filteredDs = filteredDs.filter((ds) => ds.startBlock <= processedHeight);
+        if (filteredDs.length === 0) {
+            logger.error(`Your start block is greater than the current indexed block height in your database. Either change your startBlock (project.yaml) to <= ${processedHeight}
+         or delete your database and start again from the currently specified startBlock`);
+            process.exit(1);
+        }
+        // perform filter for custom ds
+        if (!filteredDs.length) {
+            logger.error(`Did not find any datasources with associated processor`);
+            process.exit(1);
+        }
+        return filteredDs;
+    }
+    async indexBlockData({ block, logs, transactions }, dataSources, getVM) {
+        await this.indexBlockContent(block, dataSources, getVM);
+        for (const log of logs) {
+            await this.indexEvent(log, dataSources, getVM);
+        }
+        for (const tx of transactions) {
+            await this.indexExtrinsic(tx, dataSources, getVM);
+        }
+    }
+    async indexBlockContent(block, dataSources, getVM) {
+        for (const ds of dataSources) {
+            await this.indexData(common_ethereum_1.EthereumHandlerKind.Block, block, ds, getVM);
+        }
+    }
+    async indexExtrinsic(tx, dataSources, getVM) {
+        for (const ds of dataSources) {
+            await this.indexData(common_ethereum_1.EthereumHandlerKind.Call, tx, ds, getVM);
+        }
+    }
+    async indexEvent(log, dataSources, getVM) {
+        for (const ds of dataSources) {
+            await this.indexData(common_ethereum_1.EthereumHandlerKind.Event, log, ds, getVM);
+        }
+    }
+    async indexData(kind, data, ds, getVM) {
+        let vm;
+        if ((0, common_ethereum_1.isRuntimeDs)(ds)) {
+            const handlers = ds.mapping.handlers.filter((h) => h.kind === kind && FilterTypeMap[kind](data, h.filter));
+            for (const handler of handlers) {
+                vm = vm !== null && vm !== void 0 ? vm : (await getVM(ds));
+                this.nodeConfig.profiler
+                    ? await (0, node_core_1.profilerWrap)(vm.securedExec.bind(vm), 'handlerPerformance', handler.handler)(handler.handler, [data])
+                    : await vm.securedExec(handler.handler, [data]);
+            }
+        }
+        else if ((0, common_ethereum_1.isCustomDs)(ds)) {
+            const handlers = this.filterCustomDsHandlers(ds, data, ProcessorTypeMap[kind], (data, baseFilter) => {
+                switch (kind) {
+                    case common_ethereum_1.EthereumHandlerKind.Block:
+                        return block_ethereum_1.EthereumBlockWrapped.filterBlocksProcessor(data, baseFilter);
+                    case common_ethereum_1.EthereumHandlerKind.Call:
+                        return block_ethereum_1.EthereumBlockWrapped.filterTransactionsProcessor(data, baseFilter);
+                    case common_ethereum_1.EthereumHandlerKind.Event:
+                        return block_ethereum_1.EthereumBlockWrapped.filterLogsProcessor(data, baseFilter);
+                    default:
+                        throw new Error('Unsupported handler kind');
+                }
+            });
+            for (const handler of handlers) {
+                vm = vm !== null && vm !== void 0 ? vm : (await getVM(ds));
+                await this.transformAndExecuteCustomDs(ds, vm, handler, data);
+            }
+        }
+    }
+    filterCustomDsHandlers(ds, data, baseHandlerCheck, baseFilter) {
+        const plugin = this.dsProcessorService.getDsProcessor(ds);
+        return ds.mapping.handlers
+            .filter((handler) => {
+            const processor = plugin.handlerProcessors[handler.kind];
+            if (baseHandlerCheck(processor)) {
+                processor.baseFilter;
+                return baseFilter(data, processor.baseFilter);
+            }
+            return false;
+        })
+            .filter((handler) => {
+            const processor = (0, ds_processor_service_1.asSecondLayerHandlerProcessor_1_0_0)(plugin.handlerProcessors[handler.kind]);
+            try {
+                return processor.filterProcessor({
+                    filter: handler.filter,
+                    input: data,
+                    ds,
+                });
+            }
+            catch (e) {
+                logger.error(e, 'Failed to run ds processer filter.');
+                throw e;
+            }
+        });
+    }
+    async transformAndExecuteCustomDs(ds, vm, handler, data) {
+        const plugin = this.dsProcessorService.getDsProcessor(ds);
+        const assets = await this.dsProcessorService.getAssets(ds);
+        const processor = (0, ds_processor_service_1.asSecondLayerHandlerProcessor_1_0_0)(plugin.handlerProcessors[handler.kind]);
+        const transformedData = await processor
+            .transformer({
+            input: data,
+            ds,
+            api: this.api,
+            filter: handler.filter,
+            assets,
+        })
+            .catch((e) => {
+            logger.error(e, 'Failed to transform data with ds processor.');
+            throw e;
+        });
+        // We can not run this in parallel. the transformed data items may be dependent on one another.
+        // An example of this is with Acala EVM packing multiple EVM logs into a single Substrate event
+        for (const _data of transformedData) {
+            await vm.securedExec(handler.handler, [_data]);
+        }
+    }
+};
+__decorate([
+    (0, node_core_1.profiler)(yargs_1.yargsOptions.argv.profiler),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], IndexerManager.prototype, "indexBlock", null);
+IndexerManager = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [node_core_1.StoreService,
+        node_core_1.ApiService,
+        node_core_1.PoiService,
+        sequelize_1.Sequelize,
+        SubqueryProject_1.SubqueryProject,
+        node_core_1.NodeConfig,
+        sandbox_service_1.SandboxService,
+        dynamic_ds_service_1.DynamicDsService,
+        ds_processor_service_1.DsProcessorService,
+        project_service_1.ProjectService])
+], IndexerManager);
+exports.IndexerManager = IndexerManager;
+const ProcessorTypeMap = {
+    [common_ethereum_1.EthereumHandlerKind.Block]: common_ethereum_1.isBlockHandlerProcessor,
+    [common_ethereum_1.EthereumHandlerKind.Event]: common_ethereum_1.isEventHandlerProcessor,
+    [common_ethereum_1.EthereumHandlerKind.Call]: common_ethereum_1.isCallHandlerProcessor,
+};
+const FilterTypeMap = {
+    [common_ethereum_1.EthereumHandlerKind.Block]: block_ethereum_1.EthereumBlockWrapped.filterBlocksProcessor,
+    [common_ethereum_1.EthereumHandlerKind.Event]: block_ethereum_1.EthereumBlockWrapped.filterLogsProcessor,
+    [common_ethereum_1.EthereumHandlerKind.Call]: block_ethereum_1.EthereumBlockWrapped.filterTransactionsProcessor,
+};
+//# sourceMappingURL=indexer.manager.js.map

package/dist/indexer/indexer.manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.manager.js","sourceRoot":"","sources":["../../src/indexer/indexer.manager.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,yCAAiD;AACjD,4DAUgC;AAChC,gDAU0B;AAS1B,yCAAsC;AACtC,kEAA+E;AAC/E,+DAAkE;AAClE,oCAAwC;AACxC,iEAGgC;AAChC,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AAEnD,MAAM,gBAAgB,GAAG,IAAA,eAAQ,EAAC,MAAM,CAAC,CAAC;AAE1C,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAIzB,YACU,YAA0B,EAC1B,UAAsB,EACtB,UAAsB,EACtB,SAAoB,EACpB,OAAwB,EACxB,UAAsB,EACtB,cAA8B,EAC9B,gBAAkC,EAClC,kBAAsC,EACtC,cAA8B;QAT9B,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,cAAS,GAAT,SAAS,CAAW;QACpB,YAAO,GAAP,OAAO,CAAiB;QACxB,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;QAC9B,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,mBAAc,GAAd,cAAc,CAAgB;QAEtC,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAErC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IACjC,CAAC;IAGK,AAAN,KAAK,CAAC,UAAU,CACd,YAAkC;QAElC,MAAM,EAAE,WAAW,EAAE,GAAG,YAAY,CAAC;QACrC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;QAC9C,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAE9C,IAAI,aAAa,GAAG,gBAAgB,CAAC;QACrC,IAAI,YAAwB,CAAC;QAE7B,IAAI;YACF,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;YAE/D,MAAM,WAAW,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CACjD,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC,CACzD,CAAC;YAEF,MAAM,IAAI,CAAC,cAAc,CACvB,YAAY,EACZ,WAAW;YACX,4DAA4D;YAC5D,KAAK,EAAE,EAAkB,EAAE,EAAE;gBAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAClD,EAAE,EACF,IAAI,CAAC,GAAG,EACR,YAAY,CACb,CAAC;gBAEF,uCAAuC;gBACvC,EAAE,CAAC,MAAM,CACP,KAAK,EAAE,YAAoB,EAAE,IAA8B,EAAE,EAAE;oBAC7D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,uBAAuB,CAC/D;wBACE,YAAY;wBACZ,IAAI;wBACJ,UAAU,EAAE,WAAW;qBACxB,EACD,EAAE,CACH,CAAC;oBACF,+FAA+F;oBAC/F,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACxB,gBAAgB,GAAG,IAAI,CAAC;gBAC1B,CAAC,EACD,yBAAyB,CAC1B,CAAC;gBAEF,OAAO,EAAE,CAAC;YACZ,CAAC,CACF,CAAC;YAEF,MAAM,IAAI,CAAC,YAAY,CAAC,gBAAgB,CACtC;gBACE,EAAE,GAAG,EAAE,qBAAqB,EAAE,KAAK,EAAE,WAAW,EAAE;gBAClD,EAAE,GAAG,EAAE,wBAAwB,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE;aACrD,EACD,EAAE,WAAW,EAAE,EAAE,EAAE,CACpB,CAAC;YACF,0FAA0F;YAC1F,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;YAEhD,wEAAwE;YACxE,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC;YAC3D,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,2DAA2D;gBAC3D,IAAI,CAAC,IAAA,YAAK,EAAC,aAAa,EAAE,gBAAgB,CAAC,EAAE;oBAC3C,MAAM,QAAQ,GAAG,oBAAQ,CAAC,MAAM,CAC9B,WAAW,EACX,YAAY,CAAC,KAAK,CAAC,IAAI,EACvB,aAAa,EACb,MAAM,IAAI,CAAC,UAAU,CAAC,qBAAqB,EAAE,EAC7C,IAAI,CAAC,OAAO,CAAC,EAAE,CAChB,CAAC;oBACF,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC;oBAC7B,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC;oBAC9D,IAAI,CAAC,UAAU,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;oBACpD,MAAM,IAAI,CAAC,YAAY,CAAC,gBAAgB,CACtC,CAAC,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,EAC9C,EAAE,WAAW,EAAE,EAAE,EAAE,CACpB,CAAC;iBACH;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC;YACpB,MAAM,CAAC,CAAC;SACT;QAED,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC;QAElB,OAAO;YACL,gBAAgB;YAChB,aAAa;SACd,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IACnC,CAAC;IAEO,iBAAiB,CAAC,eAAuB;QAC/C,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAC1C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAC3B,MAAM,CAAC,KAAK,CACV,2DAA2D,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,CACpF,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QACD,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,eAAe,CAAC,CAAC;QACzE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAC3B,MAAM,CAAC,KAAK,CACV,0IAA0I,eAAe;yFACxE,CAClF,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QACD,+BAA+B;QAC/B,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;YACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,EAAwB,EACnD,WAA6B,EAC7B,KAAqD;QAErD,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAExD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;SAChD;QAED,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE;YAC7B,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;SACnD;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,KAAoB,EACpB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACnE;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,EAAuB,EACvB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SAC/D;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,GAAgB,EAChB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACjE;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,IAAO,EACP,IAAuC,EACvC,EAAkB,EAClB,KAAsD;QAEtD,IAAI,EAAkB,CAAC;QACvB,IAAI,IAAA,6BAAW,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,QAAQ,GAAI,EAAE,CAAC,OAAO,CAAC,QAAkC,CAAC,MAAM,CACpE,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,IAAW,EAAE,CAAC,CAAC,MAAa,CAAC,CACvE,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,EAAE,GAAG,EAAE,aAAF,EAAE,cAAF,EAAE,GAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,IAAI,CAAC,UAAU,CAAC,QAAQ;oBACtB,CAAC,CAAC,MAAM,IAAA,wBAAY,EAChB,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,EACvB,oBAAoB,EACpB,OAAO,CAAC,OAAO,CAChB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC;oBAC5B,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;aACnD;SACF;aAAM,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAC1C,EAAE,EACF,IAAI,EACJ,gBAAgB,CAAC,IAAI,CAAC,EACtB,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE;gBACnB,QAAQ,IAAI,EAAE;oBACZ,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,qCAAoB,CAAC,qBAAqB,CAC/C,IAAqB,EACrB,UAAU,CACX,CAAC;oBACJ,KAAK,qCAAmB,CAAC,IAAI;wBAC3B,OAAO,qCAAoB,CAAC,2BAA2B,CACrD,IAA2B,EAC3B,UAAU,CACX,CAAC;oBACJ,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,qCAAoB,CAAC,mBAAmB,CAC7C,IAAmB,EACnB,UAAU,CACX,CAAC;oBACJ;wBACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;iBAC/C;YACH,CAAC,CACF,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,EAAE,GAAG,EAAE,aAAF,EAAE,cAAF,EAAE,GAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,MAAM,IAAI,CAAC,2BAA2B,CAAC,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;aAC/D;SACF;IACH,CAAC;IAEO,sBAAsB,CAC5B,EAA8C,EAC9C,IAAuC,EACvC,gBAAqC,EACrC,UAGY;QAEZ,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAE1D,OAAO,EAAE,CAAC,OAAO,CAAC,QAAQ;aACvB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACzD,IAAI,gBAAgB,CAAC,SAAS,CAAC,EAAE;gBAC/B,SAAS,CAAC,UAAU,CAAC;gBAErB,OAAO,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;aAC/C;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,IAAA,0DAAmC,EACnD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CACvC,CAAC;YAEF,IAAI;gBACF,OAAO,SAAS,CAAC,eAAe,CAAC;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,KAAK,EAAE,IAAI;oBACX,EAAE;iBACH,CAAC,CAAC;aACJ;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oCAAoC,CAAC,CAAC;gBACtD,MAAM,CAAC,CAAC;aACT;QACH,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,EAA8C,EAC9C,EAAkB,EAClB,OAA2B,EAC3B,IAAuC;QAEvC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAE3D,MAAM,SAAS,GAAG,IAAA,0DAAmC,EACnD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CACvC,CAAC;QAEF,MAAM,eAAe,GAAG,MAAM,SAAS;aACpC,WAAW,CAAC;YACX,KAAK,EAAE,IAAI;YACX,EAAE;YACF,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM;SACP,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,6CAA6C,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;QACV,CAAC,CAAC,CAAC;QAEL,+FAA+F;QAC/F,+FAA+F;QAC/F,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE;YACnC,MAAM,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;SAChD;IACH,CAAC;CACF,CAAA;AA3SO;IADL,IAAA,oBAAQ,EAAC,oBAAY,CAAC,IAAI,CAAC,QAAQ,CAAC;;;;gDA+FpC;AApHU,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAMa,wBAAY;QACd,sBAAU;QACV,sBAAU;QACX,qBAAS;QACX,iCAAe;QACZ,sBAAU;QACN,gCAAc;QACZ,qCAAgB;QACd,yCAAkB;QACtB,gCAAc;GAd7B,cAAc,CAiU1B;AAjUY,wCAAc;AAyU3B,MAAM,gBAAgB,GAAG;IACvB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,wCAAsB;CACnD,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,qBAAqB;IACvE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,mBAAmB;IACrE,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,qCAAoB,CAAC,2BAA2B;CAC7E,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { hexToU8a, u8aEq } from '@polkadot/util';\nimport {\n  isBlockHandlerProcessor,\n  isCallHandlerProcessor,\n  isEventHandlerProcessor,\n  isCustomDs,\n  isRuntimeDs,\n  SubqlEthereumCustomDataSource,\n  SubqlCustomHandler,\n  EthereumHandlerKind,\n  EthereumRuntimeHandlerInputMap,\n} from '@subql/common-ethereum';\nimport {\n  ApiService,\n  PoiBlock,\n  StoreService,\n  PoiService,\n  NodeConfig,\n  getLogger,\n  profiler,\n  profilerWrap,\n  IndexerSandbox,\n} from '@subql/node-core';\nimport {\n  ApiWrapper,\n  EthereumTransaction,\n  EthereumLog,\n  SubqlRuntimeHandler,\n  EthereumBlockWrapper,\n  EthereumBlock,\n} from '@subql/types-ethereum';\nimport { Sequelize } from 'sequelize';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumBlockWrapped } from '../ethereum/block.ethereum';\nimport { yargsOptions } from '../yargs';\nimport {\n  asSecondLayerHandlerProcessor_1_0_0,\n  DsProcessorService,\n} from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\n\nconst NULL_MERKEL_ROOT = hexToU8a('0x00');\n\nconst logger = getLogger('indexer');\n\n@Injectable()\nexport class IndexerManager {\n  private api: ApiWrapper;\n  private filteredDataSources: SubqlProjectDs[];\n\n  constructor(\n    private storeService: StoreService,\n    private apiService: ApiService,\n    private poiService: PoiService,\n    private sequelize: Sequelize,\n    private project: SubqueryProject,\n    private nodeConfig: NodeConfig,\n    private sandboxService: SandboxService,\n    private dynamicDsService: DynamicDsService,\n    private dsProcessorService: DsProcessorService,\n    private projectService: ProjectService,\n  ) {\n    logger.info('indexer manager start');\n\n    this.api = this.apiService.api;\n  }\n\n  @profiler(yargsOptions.argv.profiler)\n  async indexBlock(\n    blockContent: EthereumBlockWrapper,\n  ): Promise<{ dynamicDsCreated: boolean; operationHash: Uint8Array }> {\n    const { blockHeight } = blockContent;\n    let dynamicDsCreated = false;\n    const tx = await this.sequelize.transaction();\n    this.storeService.setTransaction(tx);\n    this.storeService.setBlockHeight(blockHeight);\n\n    let operationHash = NULL_MERKEL_ROOT;\n    let poiBlockHash: Uint8Array;\n\n    try {\n      this.filteredDataSources = this.filterDataSources(blockHeight);\n\n      const datasources = this.filteredDataSources.concat(\n        ...(await this.dynamicDsService.getDynamicDatasources()),\n      );\n\n      await this.indexBlockData(\n        blockContent,\n        datasources,\n        // eslint-disable-next-line @typescript-eslint/require-await\n        async (ds: SubqlProjectDs) => {\n          const vm = this.sandboxService.getDsProcessorWrapper(\n            ds,\n            this.api,\n            blockContent,\n          );\n\n          // Inject function to create ds into vm\n          vm.freeze(\n            async (templateName: string, args?: Record<string, unknown>) => {\n              const newDs = await this.dynamicDsService.createDynamicDatasource(\n                {\n                  templateName,\n                  args,\n                  startBlock: blockHeight,\n                },\n                tx,\n              );\n              // Push the newly created dynamic ds to be processed this block on any future extrinsics/events\n              datasources.push(newDs);\n              dynamicDsCreated = true;\n            },\n            'createDynamicDatasource',\n          );\n\n          return vm;\n        },\n      );\n\n      await this.storeService.setMetadataBatch(\n        [\n          { key: 'lastProcessedHeight', value: blockHeight },\n          { key: 'lastProcessedTimestamp', value: Date.now() },\n        ],\n        { transaction: tx },\n      );\n      // Db Metadata increase BlockCount, in memory ref to block-dispatcher _processedBlockCount\n      await this.storeService.incrementBlockCount(tx);\n\n      // Need calculate operationHash to ensure correct offset insert all time\n      operationHash = this.storeService.getOperationMerkleRoot();\n      if (this.nodeConfig.proofOfIndex) {\n        //check if operation is null, then poi will not be inserted\n        if (!u8aEq(operationHash, NULL_MERKEL_ROOT)) {\n          const poiBlock = PoiBlock.create(\n            blockHeight,\n            blockContent.block.hash,\n            operationHash,\n            await this.poiService.getLatestPoiBlockHash(),\n            this.project.id,\n          );\n          poiBlockHash = poiBlock.hash;\n          await this.storeService.setPoi(poiBlock, { transaction: tx });\n          this.poiService.setLatestPoiBlockHash(poiBlockHash);\n          await this.storeService.setMetadataBatch(\n            [{ key: 'lastPoiHeight', value: blockHeight }],\n            { transaction: tx },\n          );\n        }\n      }\n    } catch (e) {\n      await tx.rollback();\n      throw e;\n    }\n\n    await tx.commit();\n\n    return {\n      dynamicDsCreated,\n      operationHash,\n    };\n  }\n\n  async start(): Promise<void> {\n    await this.projectService.init();\n  }\n\n  private filterDataSources(processedHeight: number): SubqlProjectDs[] {\n    let filteredDs = this.project.dataSources;\n    if (filteredDs.length === 0) {\n      logger.error(\n        `Did not find any dataSource match with network specName ${this.api.getSpecName()}`,\n      );\n      process.exit(1);\n    }\n    filteredDs = filteredDs.filter((ds) => ds.startBlock <= processedHeight);\n    if (filteredDs.length === 0) {\n      logger.error(\n        `Your start block is greater than the current indexed block height in your database. Either change your startBlock (project.yaml) to <= ${processedHeight}\n         or delete your database and start again from the currently specified startBlock`,\n      );\n      process.exit(1);\n    }\n    // perform filter for custom ds\n    if (!filteredDs.length) {\n      logger.error(`Did not find any datasources with associated processor`);\n      process.exit(1);\n    }\n    return filteredDs;\n  }\n\n  private async indexBlockData(\n    { block, logs, transactions }: EthereumBlockWrapper,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    await this.indexBlockContent(block, dataSources, getVM);\n\n    for (const log of logs) {\n      await this.indexEvent(log, dataSources, getVM);\n    }\n\n    for (const tx of transactions) {\n      await this.indexExtrinsic(tx, dataSources, getVM);\n    }\n  }\n\n  private async indexBlockContent(\n    block: EthereumBlock,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Block, block, ds, getVM);\n    }\n  }\n\n  private async indexExtrinsic(\n    tx: EthereumTransaction,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Call, tx, ds, getVM);\n    }\n  }\n\n  private async indexEvent(\n    log: EthereumLog,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Event, log, ds, getVM);\n    }\n  }\n\n  private async indexData<K extends EthereumHandlerKind>(\n    kind: K,\n    data: EthereumRuntimeHandlerInputMap[K],\n    ds: SubqlProjectDs,\n    getVM: (ds: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    let vm: IndexerSandbox;\n    if (isRuntimeDs(ds)) {\n      const handlers = (ds.mapping.handlers as SubqlRuntimeHandler[]).filter(\n        (h) =>\n          h.kind === kind && FilterTypeMap[kind](data as any, h.filter as any),\n      );\n\n      for (const handler of handlers) {\n        vm = vm ?? (await getVM(ds));\n        this.nodeConfig.profiler\n          ? await profilerWrap(\n              vm.securedExec.bind(vm),\n              'handlerPerformance',\n              handler.handler,\n            )(handler.handler, [data])\n          : await vm.securedExec(handler.handler, [data]);\n      }\n    } else if (isCustomDs(ds)) {\n      const handlers = this.filterCustomDsHandlers<K>(\n        ds,\n        data,\n        ProcessorTypeMap[kind],\n        (data, baseFilter) => {\n          switch (kind) {\n            case EthereumHandlerKind.Block:\n              return EthereumBlockWrapped.filterBlocksProcessor(\n                data as EthereumBlock,\n                baseFilter,\n              );\n            case EthereumHandlerKind.Call:\n              return EthereumBlockWrapped.filterTransactionsProcessor(\n                data as EthereumTransaction,\n                baseFilter,\n              );\n            case EthereumHandlerKind.Event:\n              return EthereumBlockWrapped.filterLogsProcessor(\n                data as EthereumLog,\n                baseFilter,\n              );\n            default:\n              throw new Error('Unsupported handler kind');\n          }\n        },\n      );\n\n      for (const handler of handlers) {\n        vm = vm ?? (await getVM(ds));\n        await this.transformAndExecuteCustomDs(ds, vm, handler, data);\n      }\n    }\n  }\n\n  private filterCustomDsHandlers<K extends EthereumHandlerKind>(\n    ds: SubqlEthereumCustomDataSource<string, any>,\n    data: EthereumRuntimeHandlerInputMap[K],\n    baseHandlerCheck: ProcessorTypeMap[K],\n    baseFilter: (\n      data: EthereumRuntimeHandlerInputMap[K],\n      baseFilter: any,\n    ) => boolean,\n  ): SubqlCustomHandler[] {\n    const plugin = this.dsProcessorService.getDsProcessor(ds);\n\n    return ds.mapping.handlers\n      .filter((handler) => {\n        const processor = plugin.handlerProcessors[handler.kind];\n        if (baseHandlerCheck(processor)) {\n          processor.baseFilter;\n\n          return baseFilter(data, processor.baseFilter);\n        }\n        return false;\n      })\n      .filter((handler) => {\n        const processor = asSecondLayerHandlerProcessor_1_0_0(\n          plugin.handlerProcessors[handler.kind],\n        );\n\n        try {\n          return processor.filterProcessor({\n            filter: handler.filter,\n            input: data,\n            ds,\n          });\n        } catch (e) {\n          logger.error(e, 'Failed to run ds processer filter.');\n          throw e;\n        }\n      });\n  }\n\n  private async transformAndExecuteCustomDs<K extends EthereumHandlerKind>(\n    ds: SubqlEthereumCustomDataSource<string, any>,\n    vm: IndexerSandbox,\n    handler: SubqlCustomHandler,\n    data: EthereumRuntimeHandlerInputMap[K],\n  ): Promise<void> {\n    const plugin = this.dsProcessorService.getDsProcessor(ds);\n    const assets = await this.dsProcessorService.getAssets(ds);\n\n    const processor = asSecondLayerHandlerProcessor_1_0_0(\n      plugin.handlerProcessors[handler.kind],\n    );\n\n    const transformedData = await processor\n      .transformer({\n        input: data,\n        ds,\n        api: this.api,\n        filter: handler.filter,\n        assets,\n      })\n      .catch((e) => {\n        logger.error(e, 'Failed to transform data with ds processor.');\n        throw e;\n      });\n\n    // We can not run this in parallel. the transformed data items may be dependent on one another.\n    // An example of this is with Acala EVM packing multiple EVM logs into a single Substrate event\n    for (const _data of transformedData) {\n      await vm.securedExec(handler.handler, [_data]);\n    }\n  }\n}\n\ntype ProcessorTypeMap = {\n  [EthereumHandlerKind.Block]: typeof isBlockHandlerProcessor;\n  [EthereumHandlerKind.Event]: typeof isEventHandlerProcessor;\n  [EthereumHandlerKind.Call]: typeof isCallHandlerProcessor;\n};\n\nconst ProcessorTypeMap = {\n  [EthereumHandlerKind.Block]: isBlockHandlerProcessor,\n  [EthereumHandlerKind.Event]: isEventHandlerProcessor,\n  [EthereumHandlerKind.Call]: isCallHandlerProcessor,\n};\n\nconst FilterTypeMap = {\n  [EthereumHandlerKind.Block]: EthereumBlockWrapped.filterBlocksProcessor,\n  [EthereumHandlerKind.Event]: EthereumBlockWrapped.filterLogsProcessor,\n  [EthereumHandlerKind.Call]: EthereumBlockWrapped.filterTransactionsProcessor,\n};\n"]}

package/dist/indexer/indexer.module.d.ts

@@ -0,0 +1,2 @@
+export declare class IndexerModule {
+}

package/dist/indexer/indexer.module.js

@@ -0,0 +1,52 @@
+"use strict";
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IndexerModule = void 0;
+const common_1 = require("@nestjs/common");
+const node_core_1 = require("@subql/node-core");
+const SubqueryProject_1 = require("../configure/SubqueryProject");
+const ethereum_1 = require("../ethereum");
+const dictionary_service_1 = require("./dictionary.service");
+const ds_processor_service_1 = require("./ds-processor.service");
+const dynamic_ds_service_1 = require("./dynamic-ds.service");
+const indexer_manager_1 = require("./indexer.manager");
+const project_service_1 = require("./project.service");
+const sandbox_service_1 = require("./sandbox.service");
+const worker_service_1 = require("./worker/worker.service");
+let IndexerModule = class IndexerModule {
+};
+IndexerModule = __decorate([
+    (0, common_1.Module)({
+        providers: [
+            indexer_manager_1.IndexerManager,
+            node_core_1.StoreService,
+            {
+                provide: node_core_1.ApiService,
+                useFactory: async (project) => {
+                    const apiService = new ethereum_1.EthereumApiService(project);
+                    await apiService.init();
+                    return apiService;
+                },
+                inject: [SubqueryProject_1.SubqueryProject],
+            },
+            dictionary_service_1.DictionaryService,
+            sandbox_service_1.SandboxService,
+            ds_processor_service_1.DsProcessorService,
+            dynamic_ds_service_1.DynamicDsService,
+            node_core_1.PoiService,
+            node_core_1.MmrService,
+            project_service_1.ProjectService,
+            worker_service_1.WorkerService,
+        ],
+        exports: [node_core_1.StoreService],
+    })
+], IndexerModule);
+exports.IndexerModule = IndexerModule;
+//# sourceMappingURL=indexer.module.js.map

package/dist/indexer/indexer.module.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.module.js","sourceRoot":"","sources":["../../src/indexer/indexer.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAK0B;AAC1B,kEAA+D;AAC/D,0CAAiD;AACjD,6DAAyD;AACzD,iEAA4D;AAC5D,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,4DAAwD;AA0BjD,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IAxBzB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,gCAAc;YACd,wBAAY;YACZ;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EAAE,OAAwB,EAAE,EAAE;oBAC7C,MAAM,UAAU,GAAG,IAAI,6BAAkB,CAAC,OAAO,CAAC,CAAC;oBACnD,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,iCAAe,CAAC;aAC1B;YACD,sCAAiB;YACjB,gCAAc;YACd,yCAAkB;YAClB,qCAAgB;YAChB,sBAAU;YACV,sBAAU;YACV,gCAAc;YACd,8BAAa;SACd;QACD,OAAO,EAAE,CAAC,wBAAY,CAAC;KACxB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  ApiService,\n  StoreService,\n  PoiService,\n  MmrService,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum';\nimport { DictionaryService } from './dictionary.service';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { WorkerService } from './worker/worker.service';\n\n@Module({\n  providers: [\n    IndexerManager,\n    StoreService,\n    {\n      provide: ApiService,\n      useFactory: async (project: SubqueryProject) => {\n        const apiService = new EthereumApiService(project);\n        await apiService.init();\n        return apiService;\n      },\n      inject: [SubqueryProject],\n    },\n    DictionaryService,\n    SandboxService,\n    DsProcessorService,\n    DynamicDsService,\n    PoiService,\n    MmrService,\n    ProjectService,\n    WorkerService,\n  ],\n  exports: [StoreService],\n})\nexport class IndexerModule {}\n"]}

package/dist/indexer/project.service.d.ts

@@ -0,0 +1,39 @@
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { ApiService, NodeConfig, StoreService, PoiService, MmrService } from '@subql/node-core';
+import { Sequelize } from 'sequelize';
+import { SubqueryProject } from '../configure/SubqueryProject';
+import { DsProcessorService } from './ds-processor.service';
+import { DynamicDsService } from './dynamic-ds.service';
+export declare class ProjectService {
+    private readonly dsProcessorService;
+    private readonly apiService;
+    private readonly poiService;
+    protected readonly mmrService: MmrService;
+    private readonly sequelize;
+    private readonly project;
+    private readonly storeService;
+    private readonly nodeConfig;
+    private readonly dynamicDsService;
+    private eventEmitter;
+    private _schema;
+    private metadataRepo;
+    private _startHeight;
+    private _blockOffset;
+    constructor(dsProcessorService: DsProcessorService, apiService: ApiService, poiService: PoiService, mmrService: MmrService, sequelize: Sequelize, project: SubqueryProject, storeService: StoreService, nodeConfig: NodeConfig, dynamicDsService: DynamicDsService, eventEmitter: EventEmitter2);
+    get schema(): string;
+    get blockOffset(): number;
+    get startHeight(): number;
+    private getExistingProjectSchema;
+    init(): Promise<void>;
+    private ensureProject;
+    private createProjectSchema;
+    private initDbSchema;
+    private ensureMetadata;
+    upsertMetadataBlockOffset(height: number): Promise<void>;
+    getMetadataBlockOffset(): Promise<number | undefined>;
+    getLastProcessedHeight(): Promise<number | undefined>;
+    private getStartHeight;
+    setBlockOffset(offset: number): Promise<void>;
+    getProcessedBlockCount(): Promise<number>;
+    private getStartBlockFromDataSources;
+}